Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Integrate Red Hat OpenShift AI as a model provider (#2358)

Mike Pawlowskib0773c1c19a3
diff --git a/.env.example b/.env.example
index 501d46b9f..b1dd6dfd7 100644
--- a/.env.example
+++ b/.env.example
@@ -264,6 +264,62 @@ WATSONX_PROJECT_ID=
# OPTIONAL: Delay time = base backoff seconds for the jittered exponential backoff (default 0.5).
# WATSONX_DELAY_TIME=0.5
+# --- Red Hat OpenShift AI (RHOAI) ------------------------------------------
+# Models served by KServe + vLLM inside your own OpenShift cluster, reached over
+# the OpenAI-compatible API. vLLM serves ONE model per InferenceService, so chat
+# and embeddings are two separate endpoints and both are listed below.
+#
+# The provider is hidden in every run mode as shipped. To offer it, copy
+# src/config/model_providers.yaml, set `rhoai` to true for your run mode, and
+# point OPENRAG_MODEL_PROVIDERS_CONFIG (above) at the copy. A backend restart is
+# required -- that file is not hot-reloaded.
+#
+# WARNING: these values are read only while OpenRAG has no stored config yet.
+# In the default OPENRAG_STORAGE_MODE=db, provider environment variables (these,
+# and OPENAI_API_KEY / WATSONX_* / AZURE_OPENAI_* alike) are applied exactly
+# once, when the database is first populated on the first boot; after that the
+# stored config is authoritative and the environment is not consulted again.
+# In files/hybrid mode the same happens the first time anyone saves in the
+# Settings UI (config.edited becomes true). Either way: seed BEFORE the first
+# boot, and afterwards change the provider through Settings, not the environment.
+#
+# Chat endpoint -- the predictor Service of your chat InferenceService, ending
+# in /v1. In-cluster:
+# https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1
+# RHOAI_ENDPOINT=
+#
+# Embeddings endpoint -- a second InferenceService. Leave unset only if a single
+# endpoint serves both. In-cluster:
+# https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1
+# RHOAI_EMBEDDINGS_ENDPOINT=
+#
+# Kubernetes bearer token used for BOTH endpoints. Its subject needs `get` on
+# inferenceservices in the model namespace:
+# oc create token openrag-inference -n openrag --duration=8760h
+# Put the real value in a Secret, never in a committed file.
+# RHOAI_API_KEY=
+#
+# TLS trust for the endpoints' certificates. An in-cluster Service is signed by
+# OpenShift's service-serving CA, which OpenShift projects into every pod:
+# RHOAI_TLS_VERIFY=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+# Unlike SSL_CERT_FILE this is scoped to this provider only, so it cannot break
+# OpenAI or Anthropic in the same deployment.
+# Set to `false` to skip verification. That is for ONE case -- an oc port-forward
+# to localhost, whose certificate cannot match -- and must never be set on a
+# deployed environment.
+# RHOAI_TLS_VERIFY=
+#
+# Selecting the models. The ids are each InferenceService's --served-model-name,
+# not an image tag. OpenRAG lists the real ones from each endpoint's /v1/models.
+# LLM_PROVIDER=rhoai
+# LLM_MODEL=granite-3.3-2b-instruct
+# EMBEDDING_PROVIDER=rhoai
+# EMBEDDING_MODEL=granite-embedding-english-r2
+#
+# NOTE: the chat InferenceService must be started with --enable-auto-tool-choice
+# and a --tool-call-parser (e.g. granite). OpenRAG's agent is tool-calling
+# driven, and without them vLLM rejects every tool call with a 400.
+
# LLM Provider configuration. Providers can be "anthropic", "watsonx", "ibm" or "ollama".
LLM_PROVIDER=
LLM_MODEL=
diff --git a/Makefile b/Makefile
index 20e2a4634..ab60a079a 100644
--- a/Makefile
+++ b/Makefile
@@ -364,6 +364,20 @@ help_local: ## Show local development commands
@echo " $(PURPLE)make instana-agent-up$(NC) - Start the local Instana host agent (forwards to INSTANA_AGENT_ENDPOINT)"
@echo " $(PURPLE)make instana-agent-down$(NC) - Stop the local Instana host agent"
@echo ''
+ @echo "$(PURPLE)Red Hat OpenShift AI (RHOAI) models:$(NC)"
+ @echo " Set these in $(ENV_FILE); they are exported to $(CYAN)make backend$(NC) and passed"
+ @echo " through to the backend container by $(CYAN)make dev$(NC) / $(CYAN)make dev-cpu$(NC)."
+ @echo " $(CYAN)RHOAI_ENDPOINT$(NC) - chat InferenceService URL, ending in /v1"
+ @echo " $(CYAN)RHOAI_EMBEDDINGS_ENDPOINT$(NC) - embedding InferenceService URL (vLLM serves one model each)"
+ @echo " $(CYAN)RHOAI_API_KEY$(NC) - ServiceAccount bearer token for both endpoints"
+ @echo " $(CYAN)RHOAI_TLS_VERIFY$(NC) - CA bundle path, or 'false' for a port-forward (dev only)"
+ @echo " In-cluster, use the predictor Service DNS and the projected service CA:"
+ @echo " $(CYAN)/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt$(NC)"
+ @echo " From a laptop, port-forward each InferenceService and set RHOAI_TLS_VERIFY=false:"
+ @echo " $(CYAN)oc port-forward -n <ns> svc/<isvc>-predictor 8443:8443$(NC)"
+ @echo " The provider is hidden in every run mode until $(CYAN)OPENRAG_MODEL_PROVIDERS_CONFIG$(NC)"
+ @echo " points at a model_providers.yaml enabling it (see .env.example)."
+ @echo ''
@echo "$(PURPLE)Installation:$(NC)"
@echo " $(PURPLE)make install$(NC) - Install all dependencies"
@echo " $(PURPLE)make install-be$(NC) - Install backend dependencies (uv)"
@@ -993,7 +1007,9 @@ test: ## Run all backend tests
test-unit: ## Run unit tests only
@echo "$(YELLOW)Running unit tests...$(NC)"
- uv run pytest tests/unit/ -v
+ @# A stale module that fails to import must not abort the whole suite;
+ @# the collection error is still reported and still fails the target.
+ uv run pytest tests/unit/ -v --continue-on-collection-errors
@echo "$(PURPLE)Unit tests complete.$(NC)"
test-integration: ## Run integration tests (requires infrastructure)
diff --git a/docker-compose.yml b/docker-compose.yml
index b5303facf..8c6c963c2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -87,6 +87,13 @@ services:
- WATSONX_API_KEY=${WATSONX_API_KEY}
- WATSONX_ENDPOINT=${WATSONX_ENDPOINT}
- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID}
+ # Red Hat OpenShift AI. Two endpoints because vLLM serves one model per
+ # InferenceService; RHOAI_TLS_VERIFY is a CA bundle path (or `false` for a
+ # port-forward in development only).
+ - RHOAI_ENDPOINT=${RHOAI_ENDPOINT:-}
+ - RHOAI_EMBEDDINGS_ENDPOINT=${RHOAI_EMBEDDINGS_ENDPOINT:-}
+ - RHOAI_API_KEY=${RHOAI_API_KEY:-}
+ - RHOAI_TLS_VERIFY=${RHOAI_TLS_VERIFY:-}
- OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT}
- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}
- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index 0ba30ca7e..5828a918d 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -63,6 +63,10 @@ These variables are initially set during the [application onboarding process](/i
| `WATSONX_API_KEY` | Not set | API key for the IBM watsonx.ai model provider. |
| `WATSONX_ENDPOINT` | Not set | Custom provider endpoint for the IBM watsonx.ai model provider. |
| `WATSONX_PROJECT_ID` | Not set | Project ID for the IBM watsonx.ai model provider. |
+| `RHOAI_ENDPOINT` | Not set | Chat endpoint for the Red Hat OpenShift AI model provider: the predictor Service of the chat `InferenceService`, ending in `/v1`. |
+| `RHOAI_EMBEDDINGS_ENDPOINT` | Not set | Embeddings endpoint for the Red Hat OpenShift AI model provider, a second `InferenceService`. Leave unset only if one endpoint serves both chat and embeddings. |
+| `RHOAI_API_KEY` | Not set | Kubernetes bearer token for the Red Hat OpenShift AI model provider, sent to both endpoints. Its subject needs `get` on `inferenceservices` in the model namespace. |
+| `RHOAI_TLS_VERIFY` | Not set | TLS trust for the Red Hat OpenShift AI endpoints. Set to a CA bundle path, such as the service-serving CA OpenShift projects into every pod (`/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`), or `false` to skip verification for a local `oc port-forward` only. Scoped to this provider, so it does not affect other providers. |
Newly indexed chunks keep the model name in `embedding_model` for compatibility and also store
`embedding_provider` plus the unambiguous `embedding_space_id` (`provider:model`). Retrieval queries
@@ -104,6 +108,7 @@ The shipped defaults are:
| `anthropic` | Yes | Yes | Yes |
| `azure_ai` | Yes | Yes | Yes |
| `azure` | Yes | Yes | Yes |
+| `rhoai` | No | No | No |
LiteLLM keys Azure twice, and the two are not interchangeable: `azure_ai` is
the Azure AI Foundry catalogue (Llama, Phi, Mistral, Cohere, and the gpt-5.x
@@ -111,6 +116,14 @@ entries Foundry serves), while `azure` is Azure OpenAI Service — the only one
of the two that carries the gpt-4.1 family. Offer both unless a deployment
uses just one.
+`rhoai` (Red Hat OpenShift AI) is hidden in every run mode as shipped, because
+its endpoints and token are specific to one OpenShift cluster. To offer it,
+copy `model_providers.yaml`, set `rhoai` to `true` for your run mode, point
+`OPENRAG_MODEL_PROVIDERS_CONFIG` at the copy, and restart the backend. The
+`RHOAI_*` variables above seed its credentials on first boot; the `models` and
+`embedding_models` rows on that entry are the fallback pickers until the
+cluster's own `/v1/models` listing has been read.
+
To change what a deployment offers, edit the `modes` values and restart, or
point `OPENRAG_MODEL_PROVIDERS_CONFIG` at your own YAML file. The file is read
once per process, so the restart is what applies the edit; reload the console
diff --git a/enhancements/providers/redhat/__init__.py b/enhancements/providers/redhat/__init__.py
new file mode 100644
index 000000000..f19b71dd3
--- /dev/null
+++ b/enhancements/providers/redhat/__init__.py
@@ -0,0 +1 @@
+"""Red Hat AI provider enhancements."""
diff --git a/enhancements/providers/redhat/openshift_ai.py b/enhancements/providers/redhat/openshift_ai.py
new file mode 100644
index 000000000..d153d34d8
--- /dev/null
+++ b/enhancements/providers/redhat/openshift_ai.py
@@ -0,0 +1,771 @@
+"""Red Hat OpenShift AI (RHOAI) — models served in the customer's own cluster.
+
+RHOAI serves models through `KServe` + `vLLM`, and `vLLM` speaks the OpenAI HTTP
+API. So there is no new transport here: the provider routes through LiteLLM's
+`hosted_vllm` key, the same one any self-hosted vLLM endpoint uses. What differs
+is everything around the call:
+
+- **One endpoint per model.** `vLLM` serves a single model per deployment, so a
+ RAG stack needs *two* `InferenceService`s — one for chat, one for embeddings —
+ at two different URLs. That is the whole reason this module exists rather than
+ a plain `hosted_vllm` row in `model_providers.yaml`: LiteLLM takes one
+ `api_base` per call, so something has to choose which one, and only the caller
+ knows whether it is embedding or completing. `litellm_credentials` therefore
+ takes a `kind`, which is the optional half of the enhancement contract (see
+ `credentials_for` in `enhancements/providers/registry.py`).
+- **Authentication is a Kubernetes bearer token.** An `InferenceService`
+ annotated `security.opendatahub.io/enable-auth: "true"` is fronted by a
+ `kube-rbac-proxy` that runs a `TokenReview` and then a `SubjectAccessReview`:
+ the token's subject needs `get` on `inferenceservices` in the model namespace.
+ A `ServiceAccount` token is what an operator has in hand, and it goes out as
+ an ordinary `Authorization: Bearer`, which is exactly what `api_key` becomes.
+- **Models are whatever the operator deployed**, so the catalogue ids come from
+ each endpoint's own `GET /v1/models` rather than LiteLLM's price table, which
+ knows no `hosted_vllm` models at all.
+
+The provider key is OpenRAG's own; `LITELLM_PROVIDER` is what it routes as.
+Nothing here imports OpenRAG config — `config_manager`, `model_catalog` and
+`llm_gateway` all read this module, so it has to stay a leaf.
+
+Verified end to end against an `RHOAI` 3.5.0 cluster (`vLLM 0.24.0+rhaiv.9`,
+CPU x86 runtime): model listing, chat and embeddings on both endpoints, with the
+`ServiceAccount` token and the projected service CA.
+
+Why `hosted_vllm` and not `openai_like`
+---------------------------------------
+Both are OpenAI-compatible passthrough keys, and either would carry the traffic.
+`hosted_vllm` is the one that carries the **TLS setting** with it. Verified
+against litellm 1.84 and a live cluster, both chat and embeddings:
+
+- `hosted_vllm` dispatches through `base_llm_http_handler`, which reads
+ `litellm_params["ssl_verify"]` — a per-call `ssl_verify=` is honoured.
+- `openai_like` dispatches through `OpenAILikeChatHandler` /
+ `OpenAILikeEmbeddingHandler`, neither of which mentions `ssl_verify`. The
+ kwarg is silently dropped and the call fails with
+ `ssl:True [SSLCertVerificationError: ... self-signed certificate in
+ certificate chain]` against a cluster CA — or, worse, would quietly verify
+ against the system store when the operator believed they had configured trust.
+
+That difference is the whole reason TLS trust can be a per-provider field here
+rather than the process-wide `SSL_CERT_FILE` that watsonx.ai on-prem is stuck
+with (see `enhancements/providers/watsonx/onprem.py`). Do not "simplify" this to
+`openai_like`.
+
+TLS: trusting the cluster's service-serving CA
+----------------------------------------------
+An in-cluster `Service` presents a certificate signed by OpenShift's
+service-serving CA, which is not in any container's default trust store.
+OpenShift projects that CA into every pod, so the fix is a path, not a mount::
+
+ RHOAI_TLS_VERIFY=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+
+`ssl_verify` rides in the credential bag and is scoped to this provider only —
+unlike `SSL_CERT_FILE`, which *replaces* certifi's roots process-wide and would
+break every public provider in the same deployment.
+
+`RHOAI_TLS_VERIFY=false` disables verification and exists for one case: an
+`oc port-forward` to `localhost`, where the certificate cannot match. It is
+development-only and must never be set on a deployed environment.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Mapping
+from typing import Any, Literal, NamedTuple
+from urllib.parse import urlsplit, urlunsplit
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+#: OpenRAG's key for the provider: the name in `model_providers.yaml`, in the
+#: settings payload, and in a `rhoai:<model>` id.
+PROVIDER_KEY = "rhoai"
+
+#: The LiteLLM provider it is routed as. `rhoai/<model>` is not a prefix LiteLLM
+#: can resolve, so the gateway swaps in this key when it builds the model
+#: string. See the module docstring for why this is `hosted_vllm`.
+LITELLM_PROVIDER = "hosted_vllm"
+
+#: The two kinds of call the gateway makes. `litellm_credentials` picks an
+#: endpoint from this, which is the one thing a generic provider never has to do.
+CallKind = Literal["chat", "embedding"]
+
+#: Credential form for the settings dialog. LiteLLM publishes a `hosted_vllm`
+#: form, but it asks for a single `api_base` — which is exactly the assumption
+#: that does not hold here, since chat and embeddings are separate deployments.
+CREDENTIAL_FIELDS: list[dict[str, Any]] = [
+ {
+ "key": "api_base",
+ "label": "Chat endpoint",
+ "placeholder": "https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1",
+ "tooltip": "URL of the InferenceService serving your chat model, ending in /v1. "
+ "In-cluster this is the predictor Service; from a laptop it is an oc port-forward "
+ "or a Route.",
+ "required": True,
+ "field_type": "text",
+ "options": None,
+ "default_value": None,
+ },
+ {
+ "key": "embedding_api_base",
+ "label": "Embeddings endpoint",
+ "placeholder": "https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1",
+ "tooltip": "URL of the InferenceService serving your embedding model, ending in /v1. "
+ "vLLM serves one model per deployment, so this is a second InferenceService. Leave "
+ "blank only if one endpoint serves both.",
+ "required": False,
+ "field_type": "text",
+ "options": None,
+ "default_value": None,
+ },
+ {
+ "key": "api_key",
+ "label": "ServiceAccount token",
+ "placeholder": None,
+ "tooltip": "A Kubernetes bearer token whose subject has `get` on inferenceservices in "
+ "the model namespace, e.g. `oc create token openrag-inference -n openrag`. Both "
+ "endpoints are authenticated with the same token.",
+ "required": True,
+ "field_type": "password",
+ "options": None,
+ "default_value": None,
+ },
+ {
+ "key": "ssl_verify",
+ "label": "TLS certificate authority",
+ "placeholder": "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt",
+ "tooltip": "Path to the CA bundle that signs the endpoint's certificate. In-cluster "
+ "this is OpenShift's projected service CA. Set to `false` to skip verification — "
+ "development only, for a port-forward whose certificate cannot match localhost.",
+ "required": False,
+ "field_type": "text",
+ "options": None,
+ "default_value": None,
+ },
+]
+
+#: Fields an operator fills in that are not LiteLLM kwargs. The embedding
+#: endpoint is selected *into* `api_base` per call; forwarding it as well would
+#: land it in the request body, since LiteLLM passes kwargs it does not
+#: recognise straight through.
+_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base"})
+
+#: LiteLLM kwargs that must not become request-body fields. `ssl_verify` is
+#: read for TLS *and* — on the `hosted_vllm` chat path, verified against litellm
+#: 1.84 with a capturing proxy — copied verbatim into the JSON sent to the
+#: endpoint (`{"model": ..., "messages": ..., "ssl_verify": false}`). vLLM
+#: ignores the unknown field; a stricter OpenAI-compatible server may 400 on
+#: it, and either way a client TLS setting has no business on the wire.
+#: `additional_drop_params` strips it from the body while leaving the TLS
+#: effect intact, on chat, streaming and embeddings alike.
+_BODY_DROP_PARAMS: tuple[str, ...] = ("ssl_verify",)
+
+#: Values of `ssl_verify` that mean "do not verify". Spelled out rather than
+#: parsed loosely: this switch turns off certificate checking, so an unrecognised
+#: value must fail closed (be treated as a CA path) rather than open.
+_FALSE_VALUES = frozenset({"false", "0", "no", "off"})
+_TRUE_VALUES = frozenset({"true", "1", "yes", "on"})
+
+#: TLS settings already reported. `resolve_ssl_verify` runs on every request, so
+#: without this the "verification is disabled" warning — which has to be a
+#: warning, since it is the one setting that silently weakens the deployment —
+#: would be emitted once per chat message and per embedded chunk.
+_reported_tls_settings: set[str] = set()
+
+
+def _warn_once(setting: str, message: str, **fields: Any) -> None:
+ """Report a TLS (or plain-HTTP) setting the first time this process sees it."""
+ if setting in _reported_tls_settings:
+ return
+ _reported_tls_settings.add(setting)
+ logger.warning(message, **fields)
+
+
+def forget_tls_warnings() -> None:
+ """Allow the TLS warnings to be emitted again. For tests."""
+ _reported_tls_settings.clear()
+
+
+#: The path segment every vLLM OpenAI endpoint is served under. Appended when an
+#: operator pastes the Service URL without it — the single most common cause of
+#: "works with curl, fails from OpenRAG", because curl was given the full path.
+_API_SUFFIX = "/v1"
+
+
+def normalized_api_base(value: Any) -> str:
+ """An endpoint URL as LiteLLM needs it: no trailing slash, ending in `/v1`.
+
+ Appends the suffix rather than rejecting the URL. An operator reads the
+ predictor `Service` address off `oc get svc`, which has no path, and the
+ alternative is a 404 from the endpoint's own router that says nothing about
+ what is wrong.
+ """
+ raw = str(value or "").strip().rstrip("/")
+ if not raw:
+ return ""
+ parts = urlsplit(raw)
+ path = parts.path.rstrip("/")
+ if path.endswith(_API_SUFFIX) or f"{_API_SUFFIX}/" in f"{path}/":
+ return raw
+ normalized = urlunsplit((parts.scheme, parts.netloc, f"{path}{_API_SUFFIX}", "", ""))
+ logger.debug(
+ "Appended the OpenAI path to the OpenShift AI endpoint",
+ original=raw,
+ normalized=normalized,
+ )
+ return normalized
+
+
+def resolve_ssl_verify(value: Any) -> bool | str:
+ """The TLS setting for one endpoint: `True`, `False`, or a CA bundle path.
+
+ Blank means "verify normally". A recognised false-y word disables
+ verification. Anything else is taken as a filesystem path — including a path
+ that does not exist, which is *warned about and passed through* so the call
+ fails on the missing CA instead of quietly succeeding against the system
+ trust store, which is not what the operator asked for.
+ """
+ raw = str(value if value is not None else "").strip()
+ if not raw:
+ return True
+ lowered = raw.lower()
+ if lowered in _FALSE_VALUES:
+ _warn_once(
+ "disabled",
+ "TLS verification is disabled for OpenShift AI. This is a development-only "
+ "setting for a port-forwarded endpoint; never use it on a deployed environment.",
+ )
+ return False
+ if lowered in _TRUE_VALUES:
+ return True
+
+ import os
+
+ if not os.path.exists(raw):
+ _warn_once(
+ f"missing:{raw}",
+ "The OpenShift AI CA bundle path does not exist; TLS verification will fail "
+ "until it does. In a pod the projected service CA is at "
+ "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt.",
+ path=raw,
+ )
+ return raw
+
+
+def _values(stored: Mapping[str, Any] | None) -> dict[str, str]:
+ """Credential values as trimmed strings, with blanks dropped.
+
+ Accepts either the stored form or the LiteLLM one `litellm_credentials`
+ builds from it — the health check and model discovery are handed both by
+ different callers. That is why a bool is spelled out rather than tested for
+ truth: a translated `ssl_verify=False` must come back as `"false"`, not
+ vanish as a blank, or verification is quietly switched back on. Lists
+ (`additional_drop_params`) are not credentials and are skipped.
+ """
+ values: dict[str, str] = {}
+ for name, value in (stored or {}).items():
+ if isinstance(value, bool):
+ text = "true" if value else "false"
+ elif value is None or isinstance(value, (list, tuple, dict, set)):
+ continue
+ else:
+ text = str(value).strip()
+ if text:
+ values[str(name)] = text
+ return values
+
+
+def _warn_if_cleartext(label: str, api_base: str) -> None:
+ """Flag an `http://` endpoint once: the ServiceAccount token goes out in
+ the clear on every call to it.
+
+ A warning, not a rejection. A raw-deployment predictor on the cluster
+ network with auth disabled, or an `oc port-forward` to localhost, are
+ legitimately plain HTTP — but pasting the wrong scheme for a real Service
+ is silent otherwise, and it is the strictly weaker cousin of the disabled
+ TLS verification this module already warns about.
+ """
+ if not api_base:
+ return
+ parts = urlsplit(api_base)
+ if parts.scheme.lower() != "http":
+ return
+ _warn_once(
+ f"http:{parts.netloc}",
+ "The OpenShift AI endpoint uses plain HTTP; the ServiceAccount token will be "
+ "sent in cleartext on every request. Use https:// for anything beyond a "
+ "port-forward to localhost or a cluster-local endpoint with auth disabled.",
+ endpoint=label,
+ host=parts.netloc,
+ )
+
+
+def endpoints(stored: Mapping[str, Any] | None) -> tuple[str, str]:
+ """`(chat_base, embedding_base)`, both normalized, either possibly empty.
+
+ The embedding endpoint falls back to the chat one: a deployment that serves
+ both from a single endpoint is unusual for `vLLM` but perfectly valid for
+ another OpenAI-compatible gateway fronting the cluster, and it keeps the
+ second field optional.
+ """
+ values = _values(stored)
+ chat = normalized_api_base(values.get("api_base"))
+ embedding = normalized_api_base(values.get("embedding_api_base")) or chat
+ _warn_if_cleartext("chat", chat)
+ if embedding != chat:
+ _warn_if_cleartext("embedding", embedding)
+ return chat, embedding
+
+
+def ssl_verify_for(stored: Mapping[str, Any] | None) -> bool | str:
+ """The TLS setting this module's own httpx calls should use.
+
+ Model discovery and the health check talk to the endpoint directly rather
+ than through LiteLLM, so without this they could fail on a certificate the
+ gateway accepts, or the reverse.
+ """
+ return resolve_ssl_verify(_values(stored).get("ssl_verify"))
+
+
+def litellm_credentials(stored: Mapping[str, Any], *, kind: CallKind = "chat") -> dict[str, Any]:
+ """Stored form values as LiteLLM kwargs for `hosted_vllm`, for one call kind.
+
+ This is the only place the two-endpoint shape collapses into the single
+ `api_base` LiteLLM takes. `kind` is supplied by the gateway, the health check
+ and the pre-save validator, so all three exercise the same endpoint the real
+ traffic will.
+
+ Returns nothing at all when no endpoint is configured. That is deliberate
+ and not merely tidiness: LiteLLM falls back to `HOSTED_VLLM_API_BASE` from
+ the process environment when `api_base` is absent, so handing it a bare
+ `api_key` could send the cluster's ServiceAccount token to an unrelated host
+ that happens to be configured there.
+ """
+ values = _values(stored)
+ chat_base, embedding_base = endpoints(values)
+ api_base = embedding_base if kind == "embedding" else chat_base
+ if not api_base:
+ return {}
+
+ credentials: dict[str, Any] = {
+ name: value
+ for name, value in values.items()
+ if name not in _LOCAL_ONLY_FIELDS and name not in {"api_base", "ssl_verify"}
+ }
+ credentials["api_base"] = api_base
+ credentials["ssl_verify"] = resolve_ssl_verify(values.get("ssl_verify"))
+ credentials["additional_drop_params"] = list(_BODY_DROP_PARAMS)
+ return credentials
+
+
+# --------------------------------------------------------------------------
+# Model discovery
+# --------------------------------------------------------------------------
+
+#: How long a fetched model list is reused. The set of served models changes
+#: when an administrator deploys an InferenceService, not per request.
+MODELS_TTL_SECONDS = 300
+
+#: Guard against a hung endpoint holding up a catalogue request. The catalogue
+#: has a configured fallback, so failing fast is better than waiting.
+MODELS_TIMEOUT_SECONDS = 10.0
+
+#: Same, for the pre-save health check, which a human is waiting on.
+HEALTH_TIMEOUT_SECONDS = 10.0
+
+#: Retries for the health check. Fewer than the catalogue path: one retry still
+#: rides out a pod mid-rollout, and a human is waiting on the answer.
+HEALTH_MAX_RETRIES = 1
+
+#: Upper bound on the whole pre-save check, both endpoints, retries and backoff
+#: included. Without it two unreachable endpoints × attempts × timeout adds up
+#: to a minute during which the settings save just hangs.
+HEALTH_TOTAL_BUDGET_SECONDS = 25.0
+
+#: The listing path, relative to an `api_base` that already ends in `/v1`.
+MODELS_PATH = "/models"
+
+
+class ClusterModels(NamedTuple):
+ """What the cluster is known to serve, one list per endpoint.
+
+ A half is None when that endpoint could not be asked and nothing fresh is
+ remembered for it. The catalogue then keeps its configured fallback for
+ that half alone — a rolling embedding deployment must not empty the
+ embedding picker, let alone for the whole cache TTL.
+ """
+
+ chat: tuple[str, ...] | None
+ embedding: tuple[str, ...] | None
+
+
+class _Listing(NamedTuple):
+ """One endpoint's answer and when it was fetched."""
+
+ models: tuple[str, ...]
+ at: float
+
+
+#: Each endpoint is remembered on its own clock, so a failed listing leaves
+#: that half on its previous answer (until it ages out) instead of stamping
+#: an empty list with a fresh timestamp.
+_models_cache: dict[str, Any] = {"key": None, "chat": None, "embedding": None}
+
+
+def _fresh(listing: _Listing | None) -> tuple[str, ...] | None:
+ if listing is None or time.monotonic() - listing.at > MODELS_TTL_SECONDS:
+ return None
+ return listing.models
+
+
+def cached_models() -> ClusterModels | None:
+ """The last model lists fetched from the cluster, where still fresh.
+
+ None when neither endpoint has a fresh answer; otherwise a half that has
+ aged out is None on its own, and the caller falls back for just that one.
+ """
+ models = ClusterModels(
+ chat=_fresh(_models_cache["chat"]), embedding=_fresh(_models_cache["embedding"])
+ )
+ if models.chat is None and models.embedding is None:
+ return None
+ return models
+
+
+def forget_models() -> None:
+ """Drop the cached lists. For tests and for a credential change."""
+ _models_cache.update(key=None, chat=None, embedding=None)
+
+
+def _cache_key(stored: Mapping[str, Any] | None) -> str:
+ """Identifies the endpoints and the token a cached list was fetched with."""
+ values = _values(stored)
+ chat, embedding = endpoints(values)
+ return f"{chat}|{embedding}|{values.get('api_key', '')}"
+
+
+def models_url(api_base: str) -> str:
+ """The model-listing URL on `api_base`, however the operator typed it."""
+ return f"{normalized_api_base(api_base)}{MODELS_PATH}"
+
+
+def _auth_headers(api_key: str) -> dict[str, str]:
+ return {"Authorization": f"Bearer {api_key}", "Accept": "application/json"}
+
+
+def _error_details(response: Any) -> str:
+ """The concise error an OpenAI-compatible endpoint returned."""
+ try:
+ body = response.json()
+ except (ValueError, TypeError):
+ return str(getattr(response, "text", ""))[:500]
+ if isinstance(body, Mapping):
+ error = body.get("error")
+ if isinstance(error, Mapping):
+ return str(error.get("message") or error.get("code") or "")[:500]
+ return (
+ str(body.get("message") or body.get("detail") or "")[:500]
+ or str(getattr(response, "text", ""))[:500]
+ )
+ return str(getattr(response, "text", ""))[:500]
+
+
+async def _http_request_with_retry(
+ method: str,
+ url: str,
+ *,
+ client: Any,
+ max_retries: int = 2,
+ backoff_factor: float = 0.5,
+ **kwargs: Any,
+) -> Any:
+ """Retry endpoint reads without depending on API validation helpers.
+
+ A `vLLM` pod that is still loading weights answers 503, and a `ModelCar`
+ cold start takes minutes — so a single attempt would report a healthy
+ deployment as broken during a rollout.
+ """
+ import httpx
+
+ for attempt in range(max_retries + 1):
+ try:
+ response = await client.request(method, url, **kwargs)
+ if response.status_code not in (429, 500, 502, 503, 504) or attempt == max_retries:
+ return response
+ except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as error:
+ if attempt == max_retries:
+ raise error
+ await asyncio.sleep(backoff_factor * (2**attempt))
+ raise RuntimeError(f"OpenShift AI request to {url} failed after retries")
+
+
+def model_ids(body: Any) -> tuple[str, ...]:
+ """Model ids out of one OpenAI `GET /v1/models` body.
+
+ Order is preserved and duplicates dropped. Anything that is not a list of
+ objects with an `id` yields nothing, so a proxy answering with an error page
+ leaves that half on its configured fallback rather than putting garbage in
+ a picker.
+ """
+ if not isinstance(body, Mapping):
+ return ()
+ data = body.get("data")
+ if not isinstance(data, list):
+ return ()
+ ids: list[str] = []
+ for entry in data:
+ if not isinstance(entry, Mapping):
+ continue
+ model_id = str(entry.get("id") or "").strip()
+ if model_id and model_id not in ids:
+ ids.append(model_id)
+ return tuple(ids)
+
+
+async def _list_models(client: Any, api_base: str, api_key: str) -> tuple[str, ...] | None:
+ """The models one endpoint serves, or None if it could not be asked."""
+ url = models_url(api_base)
+ logger.debug("Listing models on the OpenShift AI endpoint", url=url)
+ try:
+ response = await _http_request_with_retry(
+ "GET",
+ url,
+ client=client,
+ headers=_auth_headers(api_key),
+ timeout=MODELS_TIMEOUT_SECONDS,
+ )
+ except Exception as error:
+ logger.warning(
+ "Could not reach the OpenShift AI endpoint to list models",
+ url=url,
+ error=str(error),
+ )
+ return None
+ if response.status_code != 200:
+ logger.warning(
+ "The OpenShift AI endpoint rejected the model listing",
+ url=url,
+ status_code=response.status_code,
+ details=_error_details(response),
+ )
+ return None
+ try:
+ models = model_ids(response.json())
+ except (ValueError, TypeError) as error:
+ logger.warning(
+ "The OpenShift AI endpoint returned a model listing that could not be read",
+ url=url,
+ error=str(error),
+ )
+ return None
+ logger.debug("Listed models on the OpenShift AI endpoint", url=url, models=len(models))
+ return models
+
+
+async def fetch_models(credentials: Mapping[str, Any]) -> ClusterModels | None:
+ """Ask each configured endpoint which models it actually serves.
+
+ An `InferenceService` serves whatever its operator deployed, so this is the
+ only truthful source for the pickers — LiteLLM's bundled table has no
+ `hosted_vllm` models at all, and the `models:` rows in
+ `model_providers.yaml` are the fallback for when the cluster cannot be
+ reached.
+
+ Chat and embedding are told apart by *which endpoint answered*, which needs
+ no capability sniffing and cannot be wrong. When one endpoint serves both
+ (no separate embedding URL), its ids go into both lists: `GET /v1/models`
+ does not say what a model is for, and emptying a picker on a guess is worse
+ than offering an id that fails loudly if it is picked for the wrong job.
+
+ Each endpoint is listed and remembered on its own: only a half whose cached
+ answer has aged out is asked again, and a half that fails to answer keeps
+ the answer it had, or is None so the caller falls back for that half only.
+ Returns None when nothing is known at all, so a catalogue request never
+ fails because the cluster is unreachable; the caller keeps whatever it had.
+ """
+ import httpx
+
+ values = _values(credentials)
+ chat_base, embedding_base = endpoints(values)
+ api_key = values.get("api_key", "")
+ if not chat_base or not api_key:
+ logger.warning(
+ "Not listing models on OpenShift AI: credentials are incomplete. A chat endpoint "
+ "and a ServiceAccount token are needed; the configured model list is used "
+ "until then.",
+ has_chat_endpoint=bool(chat_base),
+ has_token=bool(api_key),
+ )
+ return None
+
+ key = _cache_key(values)
+ if _models_cache["key"] != key:
+ # Remembered for other endpoints or another token: none of it applies.
+ forget_models()
+ known = cached_models() or ClusterModels(chat=None, embedding=None)
+ if known.chat is not None and known.embedding is not None:
+ return known
+
+ logger.info(
+ "Listing models on OpenShift AI",
+ chat_endpoint=chat_base,
+ embedding_endpoint=embedding_base,
+ shared_endpoint=chat_base == embedding_base,
+ )
+ listed: dict[str, tuple[str, ...] | None] = {}
+ try:
+ async with httpx.AsyncClient(
+ verify=ssl_verify_for(values), timeout=MODELS_TIMEOUT_SECONDS
+ ) as client:
+ if chat_base == embedding_base:
+ shared = await _list_models(client, chat_base, api_key)
+ listed = {"chat": shared, "embedding": shared}
+ else:
+ if known.chat is None:
+ listed["chat"] = await _list_models(client, chat_base, api_key)
+ if known.embedding is None:
+ listed["embedding"] = await _list_models(client, embedding_base, api_key)
+ except Exception as error:
+ logger.warning(
+ "Could not list models on OpenShift AI; keeping the configured list",
+ error=str(error),
+ )
+ return cached_models()
+
+ if not any(listed.values()) and not any(known):
+ # Nothing usable from any endpoint, and nothing remembered. An empty
+ # listing is far more likely one we could not interpret than an
+ # endpoint serving nothing; emptying both pickers on that guess would
+ # be worse than showing the fallback, so nothing is cached.
+ if any(models is not None for models in listed.values()):
+ logger.warning("OpenShift AI listed no usable models; keeping the configured list")
+ else:
+ logger.warning("Could not list models on OpenShift AI; keeping the configured list")
+ return None
+
+ now = time.monotonic()
+ for half, models in listed.items():
+ # A half that failed or listed nothing keeps its previous answer and its
+ # older timestamp, so it is asked again next time rather than parked for
+ # a whole TTL — and an empty answer never replaces the configured
+ # fallback for that half.
+ if models:
+ _models_cache[half] = _Listing(models=models, at=now)
+ _models_cache["key"] = key
+
+ result = cached_models() or ClusterModels(chat=None, embedding=None)
+ logger.info(
+ "Listed models on OpenShift AI",
+ chat=len(result.chat) if result.chat is not None else "fallback",
+ embedding=len(result.embedding) if result.embedding is not None else "fallback",
+ )
+ return result
+
+
+# --------------------------------------------------------------------------
+# Health
+# --------------------------------------------------------------------------
+
+
+async def lightweight_health_check(credentials: Mapping[str, Any]) -> None:
+ """Validate connectivity and the token without selecting a model.
+
+ `GET /v1/models` needs no model, no prompt and no tokens billed, which is
+ what makes it a check the provider can pass before anything has been chosen
+ in Settings. Both endpoints are checked when they differ — an operator who
+ got the embedding URL wrong should find out here, not on the first ingest.
+
+ For that to happen the caller has to pass the *stored* form, which still
+ carries `embedding_api_base`; `validate_provider_setup(stored_credentials=...)`
+ is how it gets here. Given only the LiteLLM form, the one endpoint it was
+ narrowed to is checked.
+ """
+ import httpx
+
+ values = _values(credentials)
+ chat_base, embedding_base = endpoints(values)
+ if not chat_base:
+ raise Exception("No chat endpoint is configured for Red Hat OpenShift AI")
+ api_key = values.get("api_key", "")
+ if not api_key:
+ raise Exception(
+ "No ServiceAccount token is configured for Red Hat OpenShift AI. "
+ "Create one with `oc create token <serviceaccount> -n <namespace>`."
+ )
+
+ targets = [("chat", chat_base)]
+ if embedding_base and embedding_base != chat_base:
+ targets.append(("embeddings", embedding_base))
+
+ logger.info("Checking the OpenShift AI endpoints", endpoints=len(targets))
+ try:
+ # `asyncio.timeout` raises the builtin `TimeoutError`, not httpx's, so
+ # both are caught below.
+ async with (
+ httpx.AsyncClient(verify=ssl_verify_for(values)) as client,
+ asyncio.timeout(HEALTH_TOTAL_BUDGET_SECONDS),
+ ):
+ for label, api_base in targets:
+ await _check_endpoint(client, label, api_base, api_key)
+ except (httpx.TimeoutException, TimeoutError):
+ logger.error("OpenShift AI health check timed out")
+ raise Exception("The OpenShift AI endpoint did not respond in time") from None
+ logger.info("OpenShift AI health check passed", endpoints=len(targets))
+
+
+async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str) -> None:
+ """One endpoint's `GET /v1/models`, with a message that names the fix."""
+ import httpx
+
+ url = models_url(api_base)
+ try:
+ response = await _http_request_with_retry(
+ "GET",
+ url,
+ client=client,
+ max_retries=HEALTH_MAX_RETRIES,
+ headers=_auth_headers(api_key),
+ timeout=HEALTH_TIMEOUT_SECONDS,
+ )
+ except (httpx.TimeoutException, TimeoutError):
+ raise
+ except Exception as error:
+ logger.error(
+ "Could not reach the OpenShift AI endpoint",
+ endpoint=label,
+ url=url,
+ error=str(error),
+ )
+ raise Exception(
+ f"Could not reach the OpenShift AI {label} endpoint at {api_base}: {error}"
+ ) from error
+
+ if response.status_code == 200:
+ logger.debug("OpenShift AI endpoint answered", endpoint=label, url=url)
+ return
+
+ details = _error_details(response)
+ logger.error(
+ "The OpenShift AI endpoint rejected the request",
+ endpoint=label,
+ url=url,
+ status_code=response.status_code,
+ details=details,
+ )
+ if response.status_code in (401, 403):
+ raise Exception(
+ f"The OpenShift AI {label} endpoint rejected the token. Check that its subject "
+ "has `get` on inferenceservices in the model namespace, and that the token has "
+ f"not expired. {details}".strip()
+ )
+ if response.status_code == 404:
+ raise Exception(
+ f"The OpenShift AI {label} endpoint has no /v1/models at {api_base}. Check the "
+ f"URL points at the predictor Service and ends in /v1. {details}".strip()
+ )
+ raise Exception(
+ f"The OpenShift AI {label} endpoint returned {response.status_code}: {details}".strip()
+ )
diff --git a/enhancements/providers/registry.py b/enhancements/providers/registry.py
index dd0297c9d..e5683a1d1 100644
--- a/enhancements/providers/registry.py
+++ b/enhancements/providers/registry.py
@@ -7,12 +7,18 @@ or validation protocol.
from __future__ import annotations
+import functools
+import inspect
+from collections.abc import Mapping
from types import ModuleType
+from typing import Any
+from enhancements.providers.redhat import openshift_ai
from enhancements.providers.watsonx import onprem
_ENHANCEMENTS: dict[str, ModuleType] = {
onprem.PROVIDER_KEY: onprem,
+ openshift_ai.PROVIDER_KEY: openshift_ai,
}
@@ -42,3 +48,54 @@ def credential_field_overrides() -> dict[str, list[dict[str, object]]]:
def enhancements() -> tuple[ModuleType, ...]:
"""All registered enhancements, for generic catalogue refreshes."""
return tuple(_ENHANCEMENTS.values())
+
+
+def credentials_for(
+ enhancement: ModuleType,
+ stored: Mapping[str, Any],
+ kind: str = "chat",
+) -> dict[str, Any]:
+ """Translate stored form values into LiteLLM kwargs for one kind of call.
+
+ `kind` is the *optional* half of the enhancement contract. Most providers
+ reach one API with one set of credentials and take the stored values alone;
+ Red Hat OpenShift AI serves chat and embeddings from two separate
+ `InferenceService`s, so it has to know which endpoint the caller wants.
+
+ `kind` is passed only to a module whose `litellm_credentials` declares it,
+ so every other enhancement — and any future one written against the simpler
+ signature — keeps working unchanged. The flip side is that a module which
+ *needs* `kind` but drops it from its signature is called without it and
+ silently answers for chat; that is a contract error the module's own tests
+ have to catch, not something this function can detect.
+ """
+ translate = enhancement.litellm_credentials
+ if _accepts_kind(enhancement):
+ return dict(translate(stored, kind=kind))
+ return dict(translate(stored))
+
+
+def _accepts_kind(enhancement: ModuleType) -> bool:
+ """Whether the module's `litellm_credentials` takes a `kind` keyword.
+
+ Cached per module: the answer is fixed for the life of the process, and
+ this sits on the path of every gateway request.
+ """
+ try:
+ return "kind" in inspect.signature(enhancement.litellm_credentials).parameters
+ except (TypeError, ValueError):
+ # A C-implemented or otherwise unintrospectable callable. Assume the
+ # base contract rather than risk a TypeError on a real call.
+ _log_signature_fallback(getattr(enhancement, "__name__", str(enhancement)))
+ return False
+
+
+def _log_signature_fallback(module_name: str) -> None:
+ from utils.logging_config import get_logger
+
+ get_logger(__name__).debug(
+ "Could not inspect a provider enhancement's credential translation; "
+ "calling it without a call kind",
+ module=module_name,
+ )
diff --git a/frontend/components/icons/openshift-ai-logo.tsx b/frontend/components/icons/openshift-ai-logo.tsx
new file mode 100644
index 000000000..d85791e82
--- /dev/null
+++ b/frontend/components/icons/openshift-ai-logo.tsx
@@ -0,0 +1,20 @@
+export default function OpenShiftAILogo(props: React.SVGProps<SVGSVGElement>) {
+ return (
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="currentColor"
+ {...props}
+ >
+ <title>Red Hat OpenShift AI</title>
+ {/* The OpenShift mark: a hexagonal ring around a solid core. Drawn with
+ evenodd so the ring reads as a ring at 16px rather than filling in. */}
+ <path
+ fillRule="evenodd"
+ clipRule="evenodd"
+ d="M12 1.5 2.9 6.75v10.5L12 22.5l9.1-5.25V6.75L12 1.5Zm0 2.31 7.1 4.1v8.18l-7.1 4.1-7.1-4.1V7.91l7.1-4.1Z"
+ />
+ <circle cx="12" cy="12" r="3.35" />
+ </svg>
+ );
+}
diff --git a/frontend/components/models/model-features.tsx b/frontend/components/models/model-features.tsx
index b395ff5b9..f5df078b1 100644
--- a/frontend/components/models/model-features.tsx
+++ b/frontend/components/models/model-features.tsx
@@ -7,6 +7,7 @@ import AzureOpenAILogo from "@/components/icons/azure-openai-logo";
import IBMLogo from "@/components/icons/ibm-logo";
import OllamaLogo from "@/components/icons/ollama-logo";
import OpenAILogo from "@/components/icons/openai-logo";
+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";
import { CAPABILITY_ICONS } from "./capability-icons";
import type { CatalogModel } from "./catalog-models";
import {
@@ -35,6 +36,7 @@ const PROVIDER_LOGOS: Record<
anthropic: AnthropicLogo,
watsonx: IBMLogo,
watsonx_onprem: IBMLogo,
+ rhoai: OpenShiftAILogo,
ollama: OllamaLogo,
// Both Azure rows: `azure_ai` is Foundry, `azure` is Azure OpenAI Service,
// and Microsoft draws the two differently.
diff --git a/frontend/components/models/model-helpers.test.ts b/frontend/components/models/model-helpers.test.ts
index e56a84998..89f05c219 100644
--- a/frontend/components/models/model-helpers.test.ts
+++ b/frontend/components/models/model-helpers.test.ts
@@ -9,10 +9,13 @@
*/
import assert from "node:assert/strict";
+import type { ReactElement } from "react";
import { describe, it } from "vitest";
+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";
import {
canRemoveProvider,
EMBEDDING_PROVIDER_ORDER,
+ getModelLogo,
getProviderChrome,
isProviderConfigured,
LLM_PROVIDER_ORDER,
@@ -82,6 +85,19 @@ describe("getProviderChrome", () => {
});
});
+describe("getModelLogo", () => {
+ it("draws the OpenShift AI mark for the rhoai provider", () => {
+ // RHOAI (KServe + vLLM, PROVIDER_CHROME's `rhoai` row) is a distinct
+ // branch from the watsonx/azure/local ones above and below it, so it
+ // needs its own pin rather than relying on the neighbors.
+ const icon = getModelLogo(
+ "granite-3.1-8b-instruct",
+ "rhoai",
+ ) as ReactElement;
+ assert.equal(icon.type, OpenShiftAILogo);
+ });
+});
+
describe("isProviderConfigured", () => {
it("returns true when configured in custom or legacy fields", () => {
assert.equal(
diff --git a/frontend/components/models/model-helpers.tsx b/frontend/components/models/model-helpers.tsx
index d518180e7..a376afea2 100644
--- a/frontend/components/models/model-helpers.tsx
+++ b/frontend/components/models/model-helpers.tsx
@@ -6,6 +6,7 @@ import GenericProviderLogo from "@/components/icons/generic-provider-logo";
import IBMLogo from "@/components/icons/ibm-logo";
import OllamaLogo from "@/components/icons/ollama-logo";
import OpenAILogo from "@/components/icons/openai-logo";
+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";
/**
* A provider key as the backend names it.
@@ -24,6 +25,7 @@ export const KNOWN_PROVIDERS = [
"ollama",
"watsonx",
"watsonx_onprem",
+ "rhoai",
"azure_ai",
"azure",
"local",
@@ -163,6 +165,14 @@ const PROVIDER_CHROME: Record<string, ProviderChrome> = {
logoColor: "text-white",
logoBgColor: "bg-[#1063FE]",
},
+ // Models served by KServe + vLLM inside the customer's own OpenShift
+ // cluster. One card, though it is two InferenceServices behind the scenes.
+ rhoai: {
+ name: "Red Hat OpenShift AI",
+ logo: OpenShiftAILogo,
+ logoColor: "text-white",
+ logoBgColor: "bg-[#EE0000]",
+ },
// Microsoft draws the two Azure model services differently, and neither is
// the generic Azure logo. AiFoundryLogo paints its own gradient, so
// logoColor does nothing for it and the tile stays white.
@@ -217,6 +227,8 @@ export function getModelLogo(modelValue: string, provider?: string) {
return <OllamaLogo className="w-4 h-4" />;
} else if (provider === "watsonx" || provider === "watsonx_onprem") {
return <IBMLogo className="w-4 h-4" />;
+ } else if (provider === "rhoai") {
+ return <OpenShiftAILogo className="w-4 h-4" />;
} else if (provider === "azure") {
return <AzureOpenAILogo className="w-4 h-4" />;
} else if (provider === "azure_ai") {
diff --git a/frontend/components/models/model-selector.test.tsx b/frontend/components/models/model-selector.test.tsx
index 4ae0c6a71..6b1bb5dfd 100644
--- a/frontend/components/models/model-selector.test.tsx
+++ b/frontend/components/models/model-selector.test.tsx
@@ -1,6 +1,9 @@
+import { HttpResponse, http } from "msw";
import { describe, expect, it, vi } from "vitest";
+import { server } from "@/test-utils/msw/server";
import { renderWithProviders, screen, userEvent } from "@/test-utils/render";
import { ModelSelector } from "./model-selector";
+import type { GroupedModelOption } from "./types";
describe("ModelSelector custom entries", () => {
it("offers a typed model under every configured provider group", async () => {
@@ -95,3 +98,82 @@ describe("ModelSelector flat previews", () => {
expect(screen.getByRole("option", { name: "gpt-4.1" })).toBeInTheDocument();
});
});
+
+/**
+ * Provider badges in the group heading.
+ *
+ * The badge text comes from `GET /api/models/providers` (`config/model_providers.yaml`
+ * on the backend), keyed by provider name — the same list Settings and
+ * Onboarding already read. This pins that a provider which declares a badge
+ * gets it rendered next to its group heading, and that one which doesn't
+ * renders no badge at all.
+ */
+
+const rhoaiGroup: GroupedModelOption = {
+ group: "OpenShift AI Models",
+ provider: "rhoai",
+ options: [{ value: "granite-3.1-8b-instruct", label: "Granite 3.1 8B" }],
+};
+
+const onpremGroup: GroupedModelOption = {
+ group: "watsonx.ai On-Prem Models",
+ provider: "watsonx_onprem",
+ options: [{ value: "ibm/granite-3-8b-instruct", label: "Granite 3 8B" }],
+};
+
+function setupBadges(groupedOptions: GroupedModelOption[]) {
+ return renderWithProviders(
+ <ModelSelector
+ groupedOptions={groupedOptions}
+ value=""
+ onValueChange={vi.fn()}
+ defaultOpen
+ />,
+ );
+}
+
+function mockProviders(
+ providers: Array<{ name: string; display_name: string; badge?: string }>,
+) {
+ server.use(
+ http.get("/api/models/providers", () =>
+ HttpResponse.json({ run_mode: "oss", providers }),
+ ),
+ );
+}
+
+describe("ModelSelector provider badges", () => {
+ it("shows the badge the providers API sent for the group's provider", async () => {
+ mockProviders([
+ {
+ name: "rhoai",
+ display_name: "Red Hat OpenShift AI",
+ badge: "Tech Preview",
+ },
+ ]);
+
+ setupBadges([rhoaiGroup]);
+
+ expect(await screen.findByText("Tech Preview")).toBeInTheDocument();
+ });
+
+ it("renders no badge when the provider declares none", async () => {
+ // A second provider that *does* declare a badge is the proof the providers
+ // response has landed: group headings render synchronously from
+ // `groupedOptions`, so waiting on one of those would resolve before the
+ // query does and the negative assertion below could never fail.
+ mockProviders([
+ { name: "rhoai", display_name: "Red Hat OpenShift AI" },
+ {
+ name: "watsonx_onprem",
+ display_name: "watsonx.ai On-Prem",
+ badge: "On-prem",
+ },
+ ]);
+
+ setupBadges([rhoaiGroup, onpremGroup]);
+
+ expect(await screen.findByText("On-prem")).toBeInTheDocument();
+ expect(screen.queryByText("Tech Preview")).not.toBeInTheDocument();
+ });
+});
diff --git a/frontend/components/models/model-selector.tsx b/frontend/components/models/model-selector.tsx
index 4323ca5cc..8ba6927fa 100644
--- a/frontend/components/models/model-selector.tsx
+++ b/frontend/components/models/model-selector.tsx
@@ -2,6 +2,7 @@
import { CheckIcon, ChevronsUpDownIcon } from "lucide-react";
import { useDeferredValue, useId, useMemo, useState } from "react";
+import { useGetModelProvidersQuery } from "@/app/api/queries/useGetModelProvidersQuery";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
@@ -100,6 +101,20 @@ export function ModelSelector({
variant = "outline",
...props
}: ModelSelectorProps) {
+ // Badges come from `config/model_providers.yaml` via the providers API, so a
+ // provider that declares one gets it in the picker heading without a
+ // frontend change — the same list Settings and Onboarding already read.
+ const { data: modelProviders } = useGetModelProvidersQuery();
+ const providerBadges = useMemo(() => {
+ const badges: Record<string, string> = {};
+ for (const provider of modelProviders?.providers ?? []) {
+ if (provider.badge) {
+ badges[provider.name] = provider.badge;
+ }
+ }
+ return badges;
+ }, [modelProviders]);
+
const [open, setOpen] = useState(defaultOpen);
const [prevDefaultOpen, setPrevDefaultOpen] = useState(defaultOpen);
if (defaultOpen !== prevDefaultOpen) {
@@ -356,9 +371,9 @@ export function ModelSelector({
<div className="w-4 h-4">{group.icon}</div>
)}
<span>{group.group}</span>
- {groupProvider === "watsonx_onprem" && (
+ {groupProvider && providerBadges[groupProvider] && (
<Badge variant="secondary" className="text-xs">
- On-prem
+ {providerBadges[groupProvider]}
</Badge>
)}
</div>
diff --git a/frontend/scripts/check-diff-coverage.mjs b/frontend/scripts/check-diff-coverage.mjs
index 6405d5340..b88d5e00a 100644
--- a/frontend/scripts/check-diff-coverage.mjs
+++ b/frontend/scripts/check-diff-coverage.mjs
@@ -36,9 +36,13 @@ const COVERAGE_FILE = path.resolve("coverage/lcov.info");
/** Source files we gate on. Mirrors coverage.include in vitest.config.mts. */
const INCLUDED =
/^(app|components|contexts|hooks|lib|enhancements)\/.*\.(ts|tsx)$/;
-/** Never gate on these: tests themselves, types, and Next route plumbing. */
+/**
+ * Never gate on these: tests themselves, types, Next route plumbing, and the
+ * pure presentational SVG wrappers under components/icons/. Mirrors
+ * coverage.exclude in vitest.config.mts.
+ */
const EXCLUDED =
- /(\.test\.tsx?$|\.d\.ts$|\/(layout|error|global-error|loading)\.tsx$|\/route\.ts$)/;
+ /(\.test\.tsx?$|\.d\.ts$|\/(layout|error|global-error|loading)\.tsx$|\/route\.ts$|^components\/icons\/)/;
function git(...a) {
return execFileSync("git", a, {
diff --git a/frontend/vitest.config.mts b/frontend/vitest.config.mts
index 37b319331..ec16d5111 100644
--- a/frontend/vitest.config.mts
+++ b/frontend/vitest.config.mts
@@ -57,6 +57,8 @@ export default defineConfig({
"app/**/global-error.tsx",
"app/**/loading.tsx",
"app/**/route.ts",
+ // Pure presentational SVG wrappers, no logic to cover.
+ "components/icons/**",
],
},
},
diff --git a/src/api/provider_health.py b/src/api/provider_health.py
index e5ccfb901..579e989bd 100644
--- a/src/api/provider_health.py
+++ b/src/api/provider_health.py
@@ -77,7 +77,6 @@ async def check_provider_health(
api_key = getattr(provider_config, "api_key", None)
endpoint = getattr(provider_config, "endpoint", None)
project_id = getattr(provider_config, "project_id", None)
- credentials = current_config.providers.credential_values(provider)
# Check if this provider is used for LLM or embedding
llm_model = (
@@ -98,6 +97,19 @@ async def check_provider_health(
if model or embedding_model_override:
llm_model = model or None
embedding_model = embedding_model_override or None
+
+ # One credential set per role, because the endpoint a provider
+ # is checked against depends on which kind of call it is being
+ # checked for (Red Hat OpenShift AI serves the two from
+ # different endpoints). Each role is probed on its own below.
+ role_credentials = {
+ kind: current_config.providers.credential_values(provider, kind=kind)
+ for kind in ("chat", "embedding")
+ }
+ # The untranslated form as well: a provider enhancement's
+ # lightweight check needs every endpoint the operator entered,
+ # and each entry above has been narrowed to one of them.
+ stored_credentials = current_config.providers.stored_credentials(provider)
except ValueError:
# Provider not found in configuration
return JSONResponse(
@@ -124,8 +136,14 @@ async def check_provider_health(
embedding_endpoint = getattr(embedding_provider_config, "endpoint", None)
embedding_project_id = getattr(embedding_provider_config, "project_id", None)
embedding_model = current_config.knowledge.embedding_model
- credentials = current_config.providers.credential_values(provider)
- embedding_credentials = current_config.providers.credential_values(embedding_provider)
+ credentials = current_config.providers.credential_values(provider, kind="chat")
+ embedding_credentials = current_config.providers.credential_values(
+ embedding_provider, kind="embedding"
+ )
+ stored_credentials = current_config.providers.stored_credentials(provider)
+ embedding_stored_credentials = current_config.providers.stored_credentials(
+ embedding_provider
+ )
# Short-circuit identical concurrent polls from the provider-health
# banner so we don't fan out N watsonx round-trips per poll cycle.
@@ -185,16 +203,35 @@ async def check_provider_health(
# rather than the dedicated api_key/endpoint/project_id fields, so
# this must be forwarded or validating one from the providers page
# runs with no credentials at all.
- await validate_provider_setup(
- provider=provider,
- api_key=api_key,
- embedding_model=embedding_model,
- llm_model=llm_model,
- endpoint=endpoint,
- project_id=project_id,
- test_completion=test_completion,
- credentials=credentials,
- )
+ #
+ # One probe per role the provider is selected for. The validator
+ # tests a single model per call, so a provider that is both the
+ # LLM and the embedding provider needs two calls, each with the
+ # credentials for that role — otherwise the chat model is never
+ # checked and the response below claims it was. With no model at
+ # all, one lightweight check runs.
+ probes = [
+ (kind, model_name)
+ for kind, model_name in (("chat", llm_model), ("embedding", embedding_model))
+ if model_name
+ ] or [("chat", None)]
+ for index, (kind, model_name) in enumerate(probes):
+ # Same spacing the polled branch applies between the two
+ # watsonx tests, so back-to-back calls don't trip its rate limit.
+ if index and test_completion and provider == "watsonx":
+ logger.info("Waiting 2 seconds before WatsonX embedding test")
+ await asyncio.sleep(2)
+ await validate_provider_setup(
+ provider=provider,
+ api_key=api_key,
+ embedding_model=model_name if kind == "embedding" else None,
+ llm_model=model_name if kind == "chat" else None,
+ endpoint=endpoint,
+ project_id=project_id,
+ test_completion=test_completion,
+ credentials=role_credentials[kind],
+ stored_credentials=stored_credentials,
+ )
return JSONResponse(
{
@@ -226,6 +263,7 @@ async def check_provider_health(
project_id=project_id,
test_completion=test_completion,
credentials=credentials,
+ stored_credentials=stored_credentials,
)
except httpx.TimeoutException as e:
# Timeout means provider is busy, not misconfigured
@@ -261,6 +299,7 @@ async def check_provider_health(
project_id=embedding_project_id,
test_completion=test_completion,
credentials=embedding_credentials,
+ stored_credentials=embedding_stored_credentials,
)
except httpx.TimeoutException as e:
# Timeout means provider is busy, not misconfigured
diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py
index 3f72852b8..67e227692 100644
--- a/src/api/provider_validation.py
+++ b/src/api/provider_validation.py
@@ -3,6 +3,7 @@
import asyncio
import json
import re
+from collections.abc import Mapping
from typing import Any
from urllib.parse import urlparse
@@ -272,6 +273,8 @@ def _provider_probe_inputs(
config: Any,
provider: str,
provider_config: Any,
+ *,
+ kind: str = "chat",
) -> tuple[str | None, str | None, str | None, dict[str, str]]:
"""Return legacy fields plus the complete LiteLLM credential map.
@@ -280,11 +283,15 @@ def _provider_probe_inputs(
every LiteLLM keyword argument in ``ProvidersConfig.credentials`` instead.
Recovery probes must use both representations because they run specifically
when Langflow has hidden the provider's real failure.
+
+ ``kind`` (``"chat"`` or ``"embedding"``) selects the endpoint for a provider
+ that serves the two from different places (Red Hat OpenShift AI), so an
+ embedding probe hits the endpoint the real embedding call will.
"""
credentials: dict[str, str] = {}
credential_values = getattr(getattr(config, "providers", None), "credential_values", None)
if callable(credential_values):
- values = credential_values(provider)
+ values = credential_values(provider, kind=kind)
if isinstance(values, dict):
credentials = dict(values)
@@ -334,6 +341,7 @@ async def probe_provider_credential_error() -> str | None:
config,
provider,
provider_config,
+ kind="embedding" if embedding_model else "chat",
)
if provider == "ollama":
if not endpoint:
@@ -424,6 +432,7 @@ async def probe_embedding_error() -> str | None:
config,
provider,
provider_config,
+ kind="embedding",
)
if provider == "ollama":
if not endpoint:
@@ -662,6 +671,7 @@ async def validate_provider_setup(
project_id: str = None,
test_completion: bool = False,
credentials: dict[str, str] | None = None,
+ stored_credentials: Mapping[str, Any] | None = None,
) -> None:
"""
Validate provider setup by testing completion with tool calling and embedding.
@@ -675,6 +685,11 @@ async def validate_provider_setup(
project_id: Project ID (required for watsonx)
test_completion: If True, performs full validation with completion/embedding tests (consumes credits).
If False, performs lightweight validation (no credits consumed). Default: False.
+ credentials: LiteLLM kwargs for the provider, as the real call will be made.
+ stored_credentials: The provider's credentials as the operator entered them, untranslated.
+ Only a provider enhancement's lightweight check reads it: a provider with
+ separate chat and embedding endpoints has ``credentials`` narrowed to one
+ of them, and the check has to see both to probe both.
Raises:
Exception: If validation fails, raises the original exception with the actual error message.
@@ -729,6 +744,7 @@ async def validate_provider_setup(
endpoint=endpoint,
project_id=project_id,
credentials=supplied,
+ stored_credentials=stored_credentials,
)
return
# Full validation with completion/embedding tests (consumes credits)
@@ -758,6 +774,7 @@ async def validate_provider_setup(
endpoint=endpoint,
project_id=project_id,
credentials=supplied,
+ stored_credentials=stored_credentials,
)
logger.info(f"Validation successful for provider: {provider_lower}")
@@ -813,8 +830,14 @@ async def test_lightweight_health(
endpoint: str = None,
project_id: str = None,
credentials: dict[str, str] | None = None,
+ stored_credentials: Mapping[str, Any] | None = None,
) -> None:
- """Test provider health with lightweight check (no credits consumed)."""
+ """Test provider health with lightweight check (no credits consumed).
+
+ ``stored_credentials`` is the untranslated form and is what a provider
+ enhancement's check is given when the caller has it; ``credentials`` is the
+ LiteLLM form and the fallback. See ``validate_provider_setup``.
+ """
if provider == "openai":
await _test_openai_lightweight_health(api_key)
@@ -829,7 +852,9 @@ async def test_lightweight_health(
elif provider == "anthropic":
await _test_anthropic_lightweight_health(api_key)
elif enhancement := get_provider_enhancement(provider):
- await enhancement.lightweight_health_check(credentials or {})
+ await enhancement.lightweight_health_check(
+ stored_credentials if stored_credentials is not None else (credentials or {})
+ )
else:
raise ValueError(f"Unknown provider: {provider}")
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.py
index 773e6dd37..94e998823 100644
--- a/src/api/settings/endpoints.py
+++ b/src/api/settings/endpoints.py
@@ -14,6 +14,7 @@ from fastapi import Depends, HTTPException, Request
from fastapi.responses import JSONResponse
from api.provider_validation import (
+ get_provider_enhancement,
sanitize_provider_error_content,
validate_provider_setup,
)
@@ -463,6 +464,10 @@ async def update_settings(
# A generic provider save normally has no selected model to
# probe. Azure is the exception: both OpenAI Service and
# Foundry resource endpoints have a read-only auth probe.
+ # Provider enhancements are the other exception: each carries
+ # a model-free `lightweight_health_check`, so a bad URL or
+ # token is rejected here rather than on the first chat or
+ # ingest.
for provider, submitted in (body.provider_credentials or {}).items():
provider_key = _provider_key(provider)
credentials = current_config.providers.pending_credentials(
@@ -489,6 +494,20 @@ async def update_settings(
endpoint=credentials.get("api_base"),
credentials=credentials,
)
+ elif get_provider_enhancement(provider_key) is not None:
+ # No model is passed, so the validator runs the
+ # enhancement's lightweight check. It is given the
+ # untranslated pending form because that is the only
+ # one carrying every endpoint the operator entered —
+ # both OpenShift AI `InferenceService` URLs, not the
+ # single one `credentials` was narrowed to.
+ await validate_provider_setup(
+ provider=provider_key,
+ credentials=credentials,
+ stored_credentials=current_config.providers.pending_stored_credentials(
+ provider_key, submitted
+ ),
+ )
# Validate LLM provider if being changed
if body.llm_provider is not None or body.llm_model is not None:
@@ -539,6 +558,9 @@ async def update_settings(
endpoint=endpoint,
project_id=project_id,
credentials=credentials,
+ stored_credentials=current_config.providers.pending_stored_credentials(
+ llm_provider_key, submitted_credentials.get(llm_provider_key, {})
+ ),
)
logger.info(f"LLM provider validation successful for {llm_provider}")
@@ -572,6 +594,7 @@ async def update_settings(
credentials = current_config.providers.pending_credentials(
embedding_provider_key,
submitted_credentials.get(embedding_provider_key, {}),
+ kind="embedding",
)
api_key = credentials.get("api_key", api_key)
endpoint = credentials.get("api_base", endpoint)
@@ -594,6 +617,10 @@ async def update_settings(
endpoint=endpoint,
project_id=project_id,
credentials=credentials,
+ stored_credentials=current_config.providers.pending_stored_credentials(
+ embedding_provider_key,
+ submitted_credentials.get(embedding_provider_key, {}),
+ ),
)
logger.info(
f"Embedding provider validation successful for {embedding_provider}"
@@ -1333,7 +1360,9 @@ async def onboarding(
endpoint=getattr(embedding_provider_config, "endpoint", None),
project_id=getattr(embedding_provider_config, "project_id", None),
test_completion=True, # Full validation with completion test - ensures provider health
- credentials=current_config.providers.credential_values(embedding_provider),
+ credentials=current_config.providers.credential_values(
+ embedding_provider, kind="embedding"
+ ),
)
logger.info(
f"Embedding provider setup validation completed successfully for {embedding_provider}"
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 99562406e..0d6660b7d 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -3,6 +3,7 @@
import json
import os
import re
+from collections.abc import Mapping, Sequence
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any, Optional, Protocol
@@ -245,7 +246,18 @@ class ProvidersConfig:
}
previous.auth_method = auth_method
previous.credentials.update(clean)
- previous.configured = True
+ # Complete against the form's required fields, not merely non-empty:
+ # a submission of just `ssl_verify` must not make a provider look
+ # callable. Same gate as the environment seed.
+ required = _required_credential_keys(key)
+ previous.configured = _credentials_complete(previous.credentials, required)
+ if not previous.configured:
+ logger.warning(
+ "Model provider credentials are incomplete; it is left unconfigured "
+ "until every required field is set",
+ provider=key,
+ missing=[name for name in required if not previous.credentials.get(name)],
+ )
self.custom[key] = previous
if key == "openai":
self.openai.api_key = clean.get("api_key", self.openai.api_key)
@@ -262,9 +274,36 @@ class ProvidersConfig:
self.ollama.endpoint = clean.get("api_base", self.ollama.endpoint)
self.ollama.configured = bool(self.ollama.endpoint)
- def pending_credentials(
+ def stored_credentials(self, provider: str) -> dict[str, str]:
+ """The provider's credentials exactly as the operator entered them.
+
+ Untranslated, and therefore complete: `credential_values()` narrows a
+ multi-endpoint provider down to the one endpoint a given call needs, so
+ anything that has to see *all* of them — model discovery across an
+ OpenShift AI deployment's chat and embedding `InferenceService`s — has
+ to read the stored form instead.
+ """
+ return dict(self.custom.get(provider.strip().lower(), GenericProviderConfig()).credentials)
+
+ def pending_stored_credentials(
self, provider: str, submitted: dict[str, str] | None = None
) -> dict[str, str]:
+ """`stored_credentials()` as it would read once `submitted` is saved.
+
+ The untranslated counterpart of `pending_credentials()`, for the pre-save
+ health check of a provider whose check needs every field — both of an
+ OpenShift AI deployment's endpoints — rather than the one LiteLLM call
+ the translated form is narrowed to.
+ """
+ return {**self.stored_credentials(provider), **_clean_submitted(submitted)}
+
+ def pending_credentials(
+ self,
+ provider: str,
+ submitted: dict[str, str] | None = None,
+ *,
+ kind: str = "chat",
+ ) -> dict[str, Any]:
"""LiteLLM kwargs for `provider` as it would be once `submitted` is saved.
Validation runs before the write, so it has to reason about the union of
@@ -274,29 +313,37 @@ class ProvidersConfig:
halves have to be merged *before* the translation, or a request that
changes the API key would be validated against a stale credential built
from the old one.
+
+ `kind` selects which endpoint a multi-endpoint provider is validated
+ against, so the pre-save probe hits the same one the real call will.
"""
+ from enhancements.providers.registry import credentials_for
from enhancements.providers.registry import get as get_provider_enhancement
key = provider.strip().lower()
- clean = {
- str(name): str(value).strip()
- for name, value in (submitted or {}).items()
- if str(name).strip() and str(value).strip()
- }
+ clean = _clean_submitted(submitted)
enhancement = get_provider_enhancement(key)
if enhancement:
stored = self.custom.get(key, GenericProviderConfig()).credentials
- return enhancement.litellm_credentials({**stored, **clean})
- values = self.credential_values(key)
+ return credentials_for(enhancement, {**stored, **clean}, kind)
+ values = self.credential_values(key, kind=kind)
values.update(clean)
return values
- def credential_values(self, provider: str) -> dict[str, str]:
- """Return LiteLLM keyword arguments for a configured provider."""
+ def credential_values(self, provider: str, *, kind: str = "chat") -> dict[str, Any]:
+ """Return LiteLLM keyword arguments for a configured provider.
+
+ `kind` is `"chat"` or `"embedding"`. It matters only to a provider whose
+ two kinds of call go to different endpoints — Red Hat OpenShift AI, where
+ `vLLM` serves one model per `InferenceService` — and is ignored by every
+ other provider. Callers that need the untranslated form should use
+ `stored_credentials()`.
+ """
+ from enhancements.providers.registry import credentials_for
from enhancements.providers.registry import get as get_provider_enhancement
key = provider.strip().lower()
- custom = dict(self.custom.get(key, GenericProviderConfig()).credentials)
+ custom: dict[str, Any] = dict(self.custom.get(key, GenericProviderConfig()).credentials)
if key == "openai":
if self.openai.api_key:
custom.setdefault("api_key", self.openai.api_key)
@@ -306,7 +353,7 @@ class ProvidersConfig:
custom.setdefault("api_key", self.anthropic.api_key)
return custom
if key == "watsonx":
- legacy = {
+ legacy: dict[str, Any] = {
name: value
for name, value in {
"api_key": self.watsonx.api_key,
@@ -327,10 +374,54 @@ class ProvidersConfig:
# (cluster URL, username, API key); LiteLLM wants a ZenApiKey. The
# translation lives with the provider so the gateway, the health
# check and the validator all issue the same call.
- return enhancement.litellm_credentials(custom)
+ return credentials_for(enhancement, custom, kind)
return custom
+def _clean_submitted(submitted: dict[str, str] | None) -> dict[str, str]:
+ """Submitted credential fields, trimmed, with blank names and values dropped."""
+ return {
+ str(name): str(value).strip()
+ for name, value in (submitted or {}).items()
+ if str(name).strip() and str(value).strip()
+ }
+
+
+def _required_credential_keys(provider: str) -> tuple[str, ...]:
+ """The fields `provider`'s form marks required, per the catalogue's spec.
+
+ Empty for a provider without a spec, or when the catalogue cannot be read
+ (LiteLLM absent, or a context where `services` is not importable): callers
+ then fall back to "any credential at all", which is what `set_credentials`
+ always did.
+ """
+ try:
+ from services.model_catalog import required_field_keys
+
+ return tuple(required_field_keys(provider))
+ except Exception:
+ logger.debug(
+ "Could not read the required credential fields for a provider",
+ provider=provider,
+ exc_info=True,
+ )
+ return ()
+
+
+def _credentials_complete(stored: Mapping[str, Any], required: Sequence[str]) -> bool:
+ """Whether a stored credential set is enough to call the provider at all.
+
+ This is what `configured` means for both write paths — the environment
+ seed and a settings save — so a provider that got half its fields cannot
+ satisfy `any_configured()`, be picked as a fallback, and then be called with
+ nothing useful (or, for `hosted_vllm`, with LiteLLM falling back to an
+ `HOSTED_VLLM_API_BASE` from the environment that points somewhere else).
+ """
+ if not required:
+ return any(str(value or "").strip() for value in stored.values())
+ return all(str(stored.get(name) or "").strip() for name in required)
+
+
@dataclass
class KnowledgeConfig:
"""Knowledge/ingestion configuration."""
@@ -595,6 +686,47 @@ class ConfigManager:
credentials["api_version"] = api_version
entry["configured"] = bool(credentials.get("api_key") and credentials.get("api_base"))
+ @staticmethod
+ def _seed_custom_provider_credentials(
+ config_data: dict[str, Any],
+ provider: str,
+ credentials: dict[str, str | None],
+ *,
+ required: tuple[str, ...],
+ ) -> None:
+ """Fill a custom provider's credentials from an arbitrary field map.
+
+ The sibling `_seed_custom_provider` understands exactly `api_key` /
+ `api_base` / `api_version`, which is all Azure needs. A provider with its
+ own form — OpenShift AI carries a second endpoint URL and a TLS setting —
+ needs every field it declares, so this takes the map instead.
+
+ `required` names the fields without which the provider cannot be called
+ at all. It gates `configured` through the same `_credentials_complete`
+ check `set_credentials` applies to a settings save, so a provider that
+ got half its fields on either path cannot satisfy `any_configured()`,
+ be picked as a fallback, and be called with nothing useful.
+ """
+ supplied = {
+ name: str(value).strip()
+ for name, value in credentials.items()
+ if str(value or "").strip()
+ }
+ if not supplied:
+ return
+ custom_providers = config_data.setdefault("providers", {}).setdefault("custom", {})
+ entry = custom_providers.setdefault(provider, {})
+ stored = entry.setdefault("credentials", {})
+ stored.update(supplied)
+ entry["configured"] = _credentials_complete(stored, required)
+ if not entry["configured"]:
+ logger.warning(
+ "Environment variables for a model provider are incomplete; it is left "
+ "unconfigured until every required field is set",
+ provider=provider,
+ missing=[name for name in required if not stored.get(name)],
+ )
+
def _load_env_overrides(
self, config_data: dict[str, Any], temp_config: Optional["OpenRAGConfig"] = None
) -> None:
@@ -675,6 +807,29 @@ class ConfigManager:
config_data, "azure_ai", azure_ai_key, azure_ai_endpoint, azure_ai_version
)
+ # Red Hat OpenShift AI. Two endpoints rather than one, because vLLM
+ # serves a single model per InferenceService — see
+ # enhancements/providers/redhat/openshift_ai.py. Seeding these is what
+ # lets a Helm (`llmProviders.rhoai.*`) or operator (`spec.rhoai`) install
+ # come up configured with no human clicking through Settings, which is
+ # the point on an air-gapped cluster.
+ rhoai_endpoint = os.getenv("RHOAI_ENDPOINT")
+ rhoai_embeddings_endpoint = os.getenv("RHOAI_EMBEDDINGS_ENDPOINT")
+ rhoai_api_key = os.getenv("RHOAI_API_KEY")
+ rhoai_tls_verify = os.getenv("RHOAI_TLS_VERIFY")
+ if rhoai_endpoint or rhoai_embeddings_endpoint or rhoai_api_key or rhoai_tls_verify:
+ self._seed_custom_provider_credentials(
+ config_data,
+ "rhoai",
+ {
+ "api_base": rhoai_endpoint,
+ "embedding_api_base": rhoai_embeddings_endpoint,
+ "api_key": rhoai_api_key,
+ "ssl_verify": rhoai_tls_verify,
+ },
+ required=("api_base", "api_key"),
+ )
+
# Knowledge settings
if os.getenv("EMBEDDING_PROVIDER"):
config_data["knowledge"]["embedding_provider"] = os.getenv("EMBEDDING_PROVIDER")
diff --git a/src/config/model_providers.yaml b/src/config/model_providers.yaml
index 55d83ba4f..5f344fc96 100644
--- a/src/config/model_providers.yaml
+++ b/src/config/model_providers.yaml
@@ -114,6 +114,36 @@ providers:
embedding_models:
- ibm/slate-30m-english-rtrvr
+ # Red Hat OpenShift AI: models served by KServe + vLLM inside the customer's
+ # own OpenShift cluster. vLLM speaks the OpenAI API, so OpenRAG routes this
+ # through LiteLLM's `hosted_vllm` provider — see
+ # `enhancements/providers/redhat/openshift_ai.py`, which also explains why it
+ # is not `openai_like` (that key silently drops the per-provider TLS setting a
+ # cluster CA needs).
+ #
+ # Off in every mode as shipped. It needs two endpoint URLs and a cluster token
+ # that only the deployment knows, so it is enabled per deployment with
+ # OPENRAG_MODEL_PROVIDERS_CONFIG rather than offered to everyone.
+ #
+ # The two lists below are a *fallback*, not the catalogue. OpenRAG asks each
+ # InferenceService what it serves (`GET /v1/models`) and shows that in the
+ # pickers, so a model the cluster does not have is never offered. These rows
+ # are what it falls back to while the endpoints are unreachable or before any
+ # credentials are entered.
+ - name: rhoai
+ display_name: Red Hat OpenShift AI
+ modes:
+ oss: false
+ on_prem: false
+ saas: false
+ # Verified on an RHOAI 3.5.0 cluster running the x86 CPU vLLM runtime.
+ # These are `--served-model-name` values, not image tags: whatever the
+ # InferenceService was started with is the id that has to be sent.
+ models:
+ - granite-3.3-2b-instruct
+ embedding_models:
+ - granite-embedding-english-r2
+
- name: anthropic
display_name: Anthropic
modes:
diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.py
index 6acf8e026..530951b01 100644
--- a/src/services/llm_gateway.py
+++ b/src/services/llm_gateway.py
@@ -147,7 +147,9 @@ def legacy_embedding_provider(model: str, config: Any | None = None) -> str | No
return provider or None
-def provider_credentials(provider: str, config=None) -> dict[str, Any]:
+def provider_credentials(
+ provider: str, config=None, *, kind: Literal["chat", "embedding"] = "chat"
+) -> dict[str, Any]:
"""LiteLLM kwargs for any configured OpenRAG provider. Never logs secrets."""
cfg = config or _get_config()
key = (provider or "").strip().lower()
@@ -157,7 +159,7 @@ def provider_credentials(provider: str, config=None) -> dict[str, Any]:
raise LlmGatewayError("LLM providers are not configured", 400) from exc
if hasattr(prov, "credential_values"):
- credentials = prov.credential_values(key)
+ credentials = prov.credential_values(key, kind=kind)
else:
provider_config = getattr(prov, key, None)
if provider_config is None:
@@ -232,7 +234,7 @@ def resolve_call(
if provider is None:
provider = default_provider(kind, cfg)
name = requested
- credentials = provider_credentials(provider, cfg)
+ credentials = provider_credentials(provider, cfg, kind=kind)
# An OpenRAG provider that LiteLLM does not know by that name is routed
# under the key it aliases (`watsonx_onprem` -> `watsonx`). The OpenRAG key
# is still what the caller sees and what credentials are stored under.
diff --git a/src/services/model_catalog.py b/src/services/model_catalog.py
index 5fa6fe376..ed1575ae1 100644
--- a/src/services/model_catalog.py
+++ b/src/services/model_catalog.py
@@ -412,7 +412,9 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:
`models:` rows in `model_providers.yaml` can only ever be a guess. Once the
cluster has said what it actually has, that wins — a model it does not
serve must not sit in the picker waiting to be chosen. The configured rows
- stay as the fallback for when it cannot be reached.
+ stay as the fallback for when it cannot be reached — per list, since a
+ provider with separate chat and embedding endpoints can have an answer
+ from one and not the other.
Entries are the `_catalog()` cache key, so substituting here rebuilds the
payload rather than serving a stale one.
@@ -424,7 +426,10 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:
if hasattr(enhancement, "cached_models")
}
return tuple(
- entry._replace(models=live.chat, embedding_models=live.embedding)
+ entry._replace(
+ models=entry.models if live.chat is None else live.chat,
+ embedding_models=entry.embedding_models if live.embedding is None else live.embedding,
+ )
if (live := live_models.get(entry.name)) is not None
else entry
for entry in entries
@@ -447,7 +452,15 @@ async def refresh_live_models() -> None:
):
continue
try:
- credentials = get_openrag_config().providers.credential_values(enhancement.PROVIDER_KEY)
+ # The stored form, not the LiteLLM one: a provider whose chat and
+ # embedding endpoints differ has its credentials narrowed to one of
+ # them by `credential_values()`, and listing models needs both.
+ providers = get_openrag_config().providers
+ credentials = (
+ providers.stored_credentials(enhancement.PROVIDER_KEY)
+ if hasattr(providers, "stored_credentials")
+ else providers.credential_values(enhancement.PROVIDER_KEY)
+ )
except Exception:
logger.debug(
"Could not read credentials for %s", enhancement.PROVIDER_KEY, exc_info=True
diff --git a/tests/unit/api/test_settings_endpoints.py b/tests/unit/api/test_settings_endpoints.py
index 2702be8f0..ba72b11a4 100644
--- a/tests/unit/api/test_settings_endpoints.py
+++ b/tests/unit/api/test_settings_endpoints.py
@@ -304,6 +304,89 @@ async def test_update_settings_rejects_invalid_azure_foundry_key_before_saving()
save.assert_not_called()
+async def test_update_settings_validates_enhancement_credentials_before_saving():
+ """A credential-only save runs a provider enhancement's model-free check.
+
+ `ProviderSettingsDialog` submits nothing but `provider_credentials`, so the
+ `llm_provider` / `embedding_provider` branches never fire. Without a branch
+ of its own, a bad OpenShift AI URL or token was stored and the UI reported
+ success; the check is handed the untranslated pending form so it sees both
+ endpoints, not the one LiteLLM was narrowed to.
+ """
+ from api.settings.endpoints import update_settings
+ from config.config_manager import OpenRAGConfig
+
+ config = OpenRAGConfig.from_dict({})
+ config.edited = True
+ submitted = {
+ "api_base": "https://granite-chat.openrag.svc:8443/v1",
+ "embedding_api_base": "https://granite-embed.openrag.svc:8443/v1",
+ "api_key": "expired-token",
+ }
+ body = SettingsUpdateBody(provider_credentials={"rhoai": submitted})
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+
+ with (
+ patch("api.settings.endpoints.get_openrag_config", return_value=config),
+ patch(
+ "api.settings.endpoints.validate_provider_setup",
+ new_callable=AsyncMock,
+ side_effect=Exception("The OpenShift AI chat endpoint rejected the token"),
+ ) as validate,
+ patch("api.settings.endpoints.config_manager.save_config_file") as save,
+ ):
+ response = await update_settings(
+ body=body,
+ session_manager=AsyncMock(),
+ user=MagicMock(spec=User),
+ models_service=MagicMock(),
+ rbac=rbac,
+ )
+
+ assert response.status_code == 400
+ assert b"rejected the token" in response.body
+ validate.assert_awaited_once()
+ kwargs = validate.await_args.kwargs
+ assert kwargs["provider"] == "rhoai"
+ assert kwargs.get("llm_model") is None and kwargs.get("embedding_model") is None
+ assert kwargs["stored_credentials"] == submitted
+ save.assert_not_called()
+
+
+async def test_update_settings_skips_pre_save_check_for_litellm_only_providers():
+ """Providers without an enhancement have no model-free probe to run."""
+ from api.settings.endpoints import update_settings
+ from config.config_manager import OpenRAGConfig
+
+ config = OpenRAGConfig.from_dict({})
+ config.edited = True
+ body = SettingsUpdateBody(
+ provider_credentials={"openai_like": {"api_base": "https://llm.example", "api_key": "k"}}
+ )
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+
+ with (
+ patch("api.settings.endpoints.get_openrag_config", return_value=config),
+ patch("api.settings.endpoints.validate_provider_setup", new_callable=AsyncMock) as validate,
+ patch("api.settings.endpoints.config_manager.save_config_file"),
+ patch("api.settings.endpoints._update_langflow_global_variables", new_callable=AsyncMock),
+ ):
+ response = await update_settings(
+ body=body,
+ session_manager=AsyncMock(),
+ user=MagicMock(spec=User),
+ models_service=MagicMock(),
+ rbac=rbac,
+ )
+
+ assert getattr(response, "status_code", 200) == 200
+ validate.assert_not_awaited()
+
+
@pytest.mark.parametrize(
"provider",
["openai", "watsonx", "anthropic", "local", "ollama", "azure", "azure_ai", "openai_like"],
diff --git a/tests/unit/config/test_generic_provider_config.py b/tests/unit/config/test_generic_provider_config.py
index 881ee57f7..2cf3f1202 100644
--- a/tests/unit/config/test_generic_provider_config.py
+++ b/tests/unit/config/test_generic_provider_config.py
@@ -136,3 +136,197 @@ def test_azure_ai_foundry_env_configures_only_foundry(monkeypatch, tmp_path):
"api_base": "https://test.services.ai.azure.com/models",
}
assert "azure" not in config.providers.custom
+
+
+def _clear_rhoai_env(monkeypatch):
+ """`.env` is loaded by the root conftest, so competing values must be dropped."""
+ for name in (
+ "RHOAI_ENDPOINT",
+ "RHOAI_EMBEDDINGS_ENDPOINT",
+ "RHOAI_API_KEY",
+ "RHOAI_TLS_VERIFY",
+ "OPENAI_API_KEY",
+ "LLM_PROVIDER",
+ "EMBEDDING_PROVIDER",
+ "LLM_MODEL",
+ "EMBEDDING_MODEL",
+ ):
+ monkeypatch.delenv(name, raising=False)
+
+
+def test_rhoai_env_seeds_both_endpoints_and_the_token(monkeypatch, tmp_path):
+ """A Helm/operator install has to come up configured without a human clicking
+ through Settings, which is the point on an air-gapped cluster."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")
+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")
+
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ assert config.providers.custom["rhoai"].configured is True
+ assert config.providers.stored_credentials("rhoai") == {
+ "api_base": "https://chat.svc:8443/v1",
+ "embedding_api_base": "https://embed.svc:8443/v1",
+ "api_key": "sha256~token",
+ "ssl_verify": "/var/run/secrets/service-ca.crt",
+ }
+
+
+def test_rhoai_env_without_a_token_stays_unconfigured(monkeypatch, tmp_path):
+ """A provider that reports configured with half a credential set satisfies
+ `any_configured()` and can then be picked as a fallback and called with
+ nothing useful."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ assert config.providers.custom["rhoai"].configured is False
+
+
+def test_rhoai_tls_env_alone_reaches_a_stored_config(monkeypatch, tmp_path):
+ """A CA path mounted in a later rollout arrives as `RHOAI_TLS_VERIFY` on its
+ own, against a config that already holds the endpoint and token. It has to
+ land, and it must not disturb `configured`."""
+ import yaml
+
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ config_file = tmp_path / "config.yaml"
+ config_file.write_text(
+ yaml.safe_dump(
+ {
+ "providers": {
+ "custom": {
+ "rhoai": {
+ "credentials": {
+ "api_base": "https://chat.svc:8443/v1",
+ "api_key": "sha256~token",
+ },
+ "configured": True,
+ }
+ }
+ }
+ }
+ ),
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")
+
+ config = ConfigManager(config_file=config_file).load_config()
+
+ assert config.providers.custom["rhoai"].configured is True
+ assert config.providers.stored_credentials("rhoai") == {
+ "api_base": "https://chat.svc:8443/v1",
+ "api_key": "sha256~token",
+ "ssl_verify": "/var/run/secrets/service-ca.crt",
+ }
+
+
+def test_rhoai_env_is_ignored_once_settings_have_been_edited(monkeypatch, tmp_path):
+ """The trap worth an explicit test: the first Settings save sets
+ `config.edited` and silently freezes every environment override, which on a
+ declaratively seeded cluster looks like the variables stopped working."""
+ import yaml
+
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ config_file = tmp_path / "config.yaml"
+ config_file.write_text(yaml.safe_dump({"edited": True}), encoding="utf-8")
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")
+
+ config = ConfigManager(config_file=config_file).load_config()
+
+ assert "rhoai" not in config.providers.custom
+
+
+def test_pending_stored_credentials_overlays_the_submission_on_the_stored_form(
+ monkeypatch, tmp_path
+):
+ """The pre-save health check of a two-endpoint provider needs the raw form
+ as it will read once saved: what is stored, with the request's non-blank
+ fields on top, and nothing translated away."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ pending = config.providers.pending_stored_credentials(
+ "rhoai", {"embedding_api_base": " https://new-embed.svc:8443/v1 ", "ssl_verify": " "}
+ )
+
+ assert pending == {
+ "api_base": "https://chat.svc:8443/v1",
+ "embedding_api_base": "https://new-embed.svc:8443/v1",
+ "api_key": "sha256~token",
+ }
+ # Pending, not saved.
+ assert config.providers.stored_credentials("rhoai")["embedding_api_base"] == (
+ "https://embed.svc:8443/v1"
+ )
+
+
+def test_a_partial_save_does_not_mark_a_provider_configured():
+ """`configured` gates `any_configured()` and the fallback-provider pick, so
+ a direct API save of just `ssl_verify` must not make `rhoai` look callable:
+ `litellm_credentials()` would return `{}` for it and LiteLLM would fall back
+ to whatever `HOSTED_VLLM_API_BASE` is in the environment."""
+ config = OpenRAGConfig.from_dict({})
+
+ config.providers.set_credentials("rhoai", {"ssl_verify": "false"})
+
+ assert config.providers.custom["rhoai"].configured is False
+ assert config.providers.any_configured() is False
+ # The value is kept, so a later save that completes the set builds on it.
+ assert config.providers.stored_credentials("rhoai") == {"ssl_verify": "false"}
+
+
+def test_a_save_that_completes_the_required_fields_configures_the_provider():
+ """The check is against the merged credentials, not the one submission, so
+ a form saved in two steps ends up configured once every required field is in."""
+ config = OpenRAGConfig.from_dict({})
+
+ config.providers.set_credentials("rhoai", {"api_base": "https://chat.svc:8443/v1"})
+ assert config.providers.custom["rhoai"].configured is False
+
+ config.providers.set_credentials("rhoai", {"api_key": "sha256~token"})
+ assert config.providers.custom["rhoai"].configured is True
+
+
+def test_a_provider_without_required_fields_is_configured_by_any_credential():
+ """Providers whose spec marks nothing required keep the old "any non-empty
+ credential" rule — Anthropic's form, for one, marks neither field."""
+ config = OpenRAGConfig.from_dict({})
+
+ config.providers.set_credentials("anthropic", {"api_base": "https://proxy.example"})
+
+ assert config.providers.custom["anthropic"].configured is True
+
+
+def test_the_settings_save_and_the_env_seed_agree_on_configured(monkeypatch, tmp_path):
+ """Both write paths run the same completeness check, so a provider cannot
+ be unconfigured from the environment yet configured from the same fields
+ entered in Settings, or the reverse."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ seeded = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ saved = OpenRAGConfig.from_dict({})
+ saved.providers.set_credentials("rhoai", {"api_base": "https://chat.svc:8443/v1"})
+
+ assert seeded.providers.custom["rhoai"].configured is False
+ assert saved.providers.custom["rhoai"].configured is False
diff --git a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py
new file mode 100644
index 000000000..e6019f4a5
--- /dev/null
+++ b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py
@@ -0,0 +1,1179 @@
+"""Red Hat OpenShift AI: KServe + vLLM endpoints, routed through LiteLLM's hosted_vllm."""
+
+import time
+from typing import Any
+
+import pytest
+
+import config.model_providers as model_providers
+from config.config_manager import (
+ AnthropicConfig,
+ GenericProviderConfig,
+ OllamaConfig,
+ OpenAIConfig,
+ ProvidersConfig,
+ WatsonXConfig,
+)
+from enhancements.providers import registry
+from enhancements.providers.redhat import openshift_ai as rhoai
+from services import model_catalog
+from services.llm_gateway import provider_credentials, resolve_call, split_model_id
+
+PROVIDER = rhoai.PROVIDER_KEY
+
+CHAT_BASE = "https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1"
+EMBED_BASE = "https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1"
+TOKEN = "sha256~not-a-real-token"
+
+CHAT_MODEL = "granite-3.3-2b-instruct"
+EMBED_MODEL = "granite-embedding-english-r2"
+
+
[email protected](autouse=True)
+def _reload_provider_config():
+ model_providers.reload()
+ yield
+ model_providers.reload()
+
+
[email protected](autouse=True)
+def _forget_module_state():
+ """Model cache and TLS warnings are process-global; no test may inherit them."""
+ rhoai.forget_models()
+ rhoai.forget_tls_warnings()
+ yield
+ rhoai.forget_models()
+ rhoai.forget_tls_warnings()
+
+
+def _stored(**overrides: Any) -> dict[str, Any]:
+ values: dict[str, Any] = {
+ "api_base": CHAT_BASE,
+ "embedding_api_base": EMBED_BASE,
+ "api_key": TOKEN,
+ }
+ values.update(overrides)
+ return {name: value for name, value in values.items() if value is not None}
+
+
+def _providers(**credentials: Any) -> ProvidersConfig:
+ return ProvidersConfig(
+ openai=OpenAIConfig(),
+ anthropic=AnthropicConfig(),
+ watsonx=WatsonXConfig(),
+ ollama=OllamaConfig(),
+ custom={PROVIDER: GenericProviderConfig(credentials=dict(credentials), configured=True)},
+ )
+
+
+def _enable_in_oss(tmp_path, monkeypatch) -> None:
+ """Point the run mode at a config file that offers this provider.
+
+ The shipped `model_providers.yaml` has every mode false — the endpoints and
+ token are deployment-specific, so it is turned on per deployment. A test that
+ needs the catalogue has to do the same thing an operator does.
+ """
+ config = tmp_path / "model_providers.yaml"
+ config.write_text(
+ "providers:\n"
+ f" - name: {PROVIDER}\n"
+ " display_name: Red Hat OpenShift AI\n"
+ " modes:\n"
+ " oss: true\n"
+ f" models:\n - {CHAT_MODEL}\n"
+ f" embedding_models:\n - {EMBED_MODEL}\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
+ monkeypatch.setenv("OPENRAG_MODEL_PROVIDERS_CONFIG", str(config))
+ model_providers.reload()
+
+
+# ---------------------------------------------------------------------------
+# Credential form
+# ---------------------------------------------------------------------------
+
+
+def test_the_form_asks_for_two_endpoints_a_token_and_a_ca() -> None:
+ """vLLM serves one model per InferenceService, so one endpoint is not enough."""
+ fields = {field["key"]: field for field in model_catalog.credential_fields(PROVIDER)}
+
+ assert set(fields) == {"api_base", "embedding_api_base", "api_key", "ssl_verify"}
+ assert fields["api_base"]["field_type"] == "text"
+ assert fields["embedding_api_base"]["field_type"] == "text"
+ assert fields["api_key"]["field_type"] == "password"
+ assert fields["ssl_verify"]["field_type"] == "text"
+
+
+def test_only_the_token_is_encrypted_at_rest() -> None:
+ """`field_type` drives encryption. A CA path is not a secret; a token is."""
+ assert model_catalog.secret_field_keys(PROVIDER) == {"api_key"}
+
+
+def test_an_endpoint_and_a_token_are_both_required() -> None:
+ assert model_catalog.required_field_keys(PROVIDER) == ["api_base", "api_key"]
+
+
+# ---------------------------------------------------------------------------
+# Endpoint selection — the reason this module exists
+# ---------------------------------------------------------------------------
+
+
+def test_a_chat_call_goes_to_the_chat_endpoint() -> None:
+ assert rhoai.litellm_credentials(_stored(), kind="chat")["api_base"] == CHAT_BASE
+
+
+def test_an_embedding_call_goes_to_the_embedding_endpoint() -> None:
+ """The whole point: one credential bag, two InferenceServices."""
+ assert rhoai.litellm_credentials(_stored(), kind="embedding")["api_base"] == EMBED_BASE
+
+
+def test_chat_is_the_default_kind() -> None:
+ """An older caller that does not know about kinds must still reach a model."""
+ assert rhoai.litellm_credentials(_stored())["api_base"] == CHAT_BASE
+
+
+def test_one_endpoint_serves_both_when_no_embedding_url_is_given() -> None:
+ """The second field is optional, so a single-endpoint gateway still works."""
+ stored = _stored(embedding_api_base=None)
+
+ assert rhoai.litellm_credentials(stored, kind="chat")["api_base"] == CHAT_BASE
+ assert rhoai.litellm_credentials(stored, kind="embedding")["api_base"] == CHAT_BASE
+
+
[email protected]("kind", ["chat", "embedding"])
+def test_the_embedding_url_never_reaches_litellm(kind) -> None:
+ """LiteLLM forwards kwargs it does not recognise, so a stray field lands in the body."""
+ assert "embedding_api_base" not in rhoai.litellm_credentials(_stored(), kind=kind)
+
+
+def test_no_endpoint_means_no_credentials_at_all() -> None:
+ """Not tidiness: LiteLLM falls back to HOSTED_VLLM_API_BASE from the process
+ environment when `api_base` is absent, so returning a bare token could send a
+ cluster ServiceAccount token to whatever unrelated host is configured there.
+ """
+ assert rhoai.litellm_credentials({"api_key": TOKEN}) == {}
+
+
+def test_the_token_is_carried_through_as_the_api_key() -> None:
+ """RHOAI's kube-rbac-proxy wants `Authorization: Bearer <token>`, which is
+ exactly what LiteLLM does with `api_key`."""
+ assert rhoai.litellm_credentials(_stored())["api_key"] == TOKEN
+
+
+def test_the_tls_setting_is_kept_off_the_wire() -> None:
+ """LiteLLM reads `ssl_verify` for TLS and, on the hosted_vllm chat path, also
+ copies it into the request body. `additional_drop_params` is how the bag
+ tells LiteLLM to strip it again."""
+ credentials = rhoai.litellm_credentials(_stored(ssl_verify="false"))
+
+ assert credentials["ssl_verify"] is False
+ assert "ssl_verify" in credentials["additional_drop_params"]
+
+
+async def test_litellm_does_not_leak_the_tls_setting_into_the_request_body() -> None:
+ """The real thing, against a local OpenAI-compatible stub: found with a
+ capturing proxy in front of a live cluster, where the body arrived as
+ `{"model": ..., "messages": ..., "ssl_verify": false}`. vLLM ignored it; a
+ stricter server would not."""
+ import json
+ import threading
+ from http.server import BaseHTTPRequestHandler, HTTPServer
+
+ import litellm
+
+ bodies: list[dict[str, Any]] = []
+
+ class _Stub(BaseHTTPRequestHandler):
+ def log_message(self, *args: Any) -> None:
+ pass
+
+ def do_POST(self) -> None:
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ bodies.append(body)
+ if self.path.endswith("/embeddings"):
+ reply = {
+ "object": "list",
+ "model": body["model"],
+ "data": [{"index": 0, "object": "embedding", "embedding": [0.0] * 4}],
+ "usage": {"prompt_tokens": 1, "total_tokens": 1},
+ }
+ else:
+ reply = {
+ "id": "chatcmpl-stub",
+ "object": "chat.completion",
+ "model": body["model"],
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "stop",
+ "message": {"role": "assistant", "content": "ok"},
+ }
+ ],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ }
+ raw = json.dumps(reply).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(raw)))
+ self.end_headers()
+ self.wfile.write(raw)
+
+ server = HTTPServer(("127.0.0.1", 0), _Stub)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ try:
+ base = f"http://127.0.0.1:{server.server_port}/v1"
+ stored = _stored(api_base=base, embedding_api_base=base, ssl_verify="false")
+ await litellm.acompletion(
+ model=f"hosted_vllm/{CHAT_MODEL}",
+ messages=[{"role": "user", "content": "hi"}],
+ drop_params=True,
+ **rhoai.litellm_credentials(stored, kind="chat"),
+ )
+ await litellm.aembedding(
+ model=f"hosted_vllm/{EMBED_MODEL}",
+ input=["hi"],
+ **rhoai.litellm_credentials(stored, kind="embedding"),
+ )
+ finally:
+ server.shutdown()
+
+ assert len(bodies) == 2
+ for body in bodies:
+ assert "ssl_verify" not in body
+ assert "additional_drop_params" not in body
+ assert "embedding_api_base" not in body
+
+
+# ---------------------------------------------------------------------------
+# Endpoint URL normalization
+# ---------------------------------------------------------------------------
+
+
+ "given, expected",
+ [
+ # `oc get svc` prints an address with no path; pasting it is the common case.
+ ("https://chat.svc:8443", "https://chat.svc:8443/v1"),
+ ("https://chat.svc:8443/", "https://chat.svc:8443/v1"),
+ ("https://chat.svc:8443/v1", "https://chat.svc:8443/v1"),
+ ("https://chat.svc:8443/v1/", "https://chat.svc:8443/v1"),
+ # An endpoint behind a path-routing proxy keeps its prefix.
+ ("https://gw.example.com/openrag/v1", "https://gw.example.com/openrag/v1"),
+ ("", ""),
+ (None, ""),
+ ],
+)
+def test_the_openai_path_is_added_but_never_doubled(given, expected) -> None:
+ assert rhoai.normalized_api_base(given) == expected
+
+
+def test_a_pasted_service_address_still_reaches_the_models_route() -> None:
+ assert rhoai.models_url("https://chat.svc:8443") == "https://chat.svc:8443/v1/models"
+
+
+# ---------------------------------------------------------------------------
+# TLS
+# ---------------------------------------------------------------------------
+
+
+def test_no_ca_configured_verifies_normally() -> None:
+ assert rhoai.resolve_ssl_verify("") is True
+ assert rhoai.resolve_ssl_verify(None) is True
+
+
[email protected]("value", ["false", "False", "0", "no", "off", " FALSE "])
+def test_verification_can_be_turned_off_for_a_port_forward(value) -> None:
+ assert rhoai.resolve_ssl_verify(value) is False
+
+
[email protected]("value", ["true", "1", "yes", "on"])
+def test_an_explicit_true_verifies_normally(value) -> None:
+ assert rhoai.resolve_ssl_verify(value) is True
+
+
+def test_a_ca_bundle_path_is_passed_through(tmp_path) -> None:
+ ca = tmp_path / "service-ca.crt"
+ ca.write_text("-----BEGIN CERTIFICATE-----", encoding="utf-8")
+
+ assert rhoai.resolve_ssl_verify(str(ca)) == str(ca)
+
+
+def _recorded_warnings(monkeypatch) -> list[str]:
+ """Warnings the module emits. Read off the module logger rather than caplog:
+ logging goes through structlog here, which does not route to caplog's handler.
+ """
+ messages: list[str] = []
+
+ class _Recorder:
+ def warning(self, message, **fields):
+ messages.append(message)
+
+ def __getattr__(self, _name):
+ return lambda *args, **kwargs: None
+
+ monkeypatch.setattr(rhoai, "logger", _Recorder())
+ return messages
+
+
+def test_a_missing_ca_path_is_warned_about_and_still_used(monkeypatch) -> None:
+ """Falling back to the system trust store would verify against roots the
+ operator never asked for, and succeed — hiding the misconfiguration. Better
+ to fail on the missing file.
+ """
+ warnings = _recorded_warnings(monkeypatch)
+ missing = "/no/such/service-ca.crt"
+
+ assert rhoai.resolve_ssl_verify(missing) == missing
+ assert any("CA bundle path does not exist" in message for message in warnings)
+
+
+def test_the_disabled_tls_warning_is_not_repeated_per_request(monkeypatch) -> None:
+ """`litellm_credentials` runs on every chat message and every embedded chunk."""
+ warnings = _recorded_warnings(monkeypatch)
+
+ for _ in range(5):
+ rhoai.resolve_ssl_verify("false")
+
+ disabled = [m for m in warnings if "TLS verification is disabled" in m]
+ assert len(disabled) == 1
+
+
+def test_a_plain_http_endpoint_is_warned_about_but_still_used(monkeypatch) -> None:
+ """The token goes out in the clear, which deserves the same treatment as
+ disabled verification — a warning, not a rejection, since a port-forward
+ and a cluster-local raw deployment are legitimately plain HTTP."""
+ warnings = _recorded_warnings(monkeypatch)
+
+ chat, embedding = rhoai.endpoints(
+ _stored(api_base="http://chat.svc:8080/v1", embedding_api_base="http://embed.svc:8080")
+ )
+
+ assert chat == "http://chat.svc:8080/v1"
+ assert embedding == "http://embed.svc:8080/v1"
+ cleartext = [m for m in warnings if "plain HTTP" in m]
+ assert len(cleartext) == 2 # one per distinct endpoint
+
+
+def test_the_cleartext_warning_is_not_repeated_per_request(monkeypatch) -> None:
+ """`endpoints()` runs inside `litellm_credentials`, i.e. on every chat
+ message and every embedded chunk."""
+ warnings = _recorded_warnings(monkeypatch)
+
+ for _ in range(5):
+ rhoai.litellm_credentials(_stored(api_base="http://chat.svc:8080/v1"))
+
+ cleartext = [m for m in warnings if "plain HTTP" in m]
+ assert len(cleartext) == 1
+
+
+def test_https_endpoints_draw_no_cleartext_warning(monkeypatch) -> None:
+ warnings = _recorded_warnings(monkeypatch)
+
+ rhoai.endpoints(_stored())
+
+ assert not [m for m in warnings if "plain HTTP" in m]
+
+
+def test_the_tls_setting_rides_in_the_credential_bag(tmp_path) -> None:
+ """Per-provider, unlike SSL_CERT_FILE, which would replace certifi's roots
+ process-wide and break every public provider in the same deployment."""
+ ca = tmp_path / "service-ca.crt"
+ ca.write_text("-----BEGIN CERTIFICATE-----", encoding="utf-8")
+
+ credentials = rhoai.litellm_credentials(_stored(ssl_verify=str(ca)))
+
+ assert credentials["ssl_verify"] == str(ca)
+ assert rhoai.litellm_credentials(_stored(ssl_verify="false"))["ssl_verify"] is False
+
+
+def test_a_translated_false_is_not_mistaken_for_a_blank() -> None:
+ """The health check and model discovery are handed the LiteLLM form by some
+ callers, where `ssl_verify` is already a bool. `False` is falsy, and a
+ truth-test on the value used to drop it — turning "do not verify" back
+ into "verify" for the one setting where that inverts the operator's intent."""
+ assert rhoai._values({"ssl_verify": False})["ssl_verify"] == "false"
+ assert rhoai._values({"ssl_verify": True})["ssl_verify"] == "true"
+ # Not a credential; must not be stringified into one.
+ assert "additional_drop_params" not in rhoai._values({"additional_drop_params": ["x"]})
+
+
+def test_verification_stays_off_when_the_health_check_gets_the_litellm_form() -> None:
+ """The regression: `RHOAI_TLS_VERIFY=false` for a port-forward, and the
+ pre-save check verifying anyway while real traffic did not."""
+ translated = rhoai.litellm_credentials(_stored(ssl_verify="false"), kind="chat")
+
+ assert translated["ssl_verify"] is False
+ assert rhoai.ssl_verify_for(translated) is False
+
+
+# ---------------------------------------------------------------------------
+# Routing
+# ---------------------------------------------------------------------------
+
+
+def test_it_routes_as_hosted_vllm() -> None:
+ """`rhoai/<model>` is not a prefix LiteLLM can resolve. `hosted_vllm` is
+ chosen over `openai_like` because only it threads a per-call `ssl_verify` —
+ see the module docstring.
+ """
+ assert model_catalog.litellm_provider_key(PROVIDER) == "hosted_vllm"
+ assert rhoai.LITELLM_PROVIDER == "hosted_vllm"
+
+
+def test_an_alias_is_a_provider_the_gateway_can_resolve() -> None:
+ assert model_catalog.is_known_provider(PROVIDER)
+
+
+def test_its_model_ids_carry_the_openrag_provider_tag() -> None:
+ assert split_model_id(f"{PROVIDER}:{CHAT_MODEL}") == (PROVIDER, CHAT_MODEL)
+ assert model_catalog.public_model_id(PROVIDER, CHAT_MODEL) == f"{PROVIDER}:{CHAT_MODEL}"
+
+
+def test_a_chat_call_resolves_to_the_chat_endpoint() -> None:
+ config = _config(_providers(**_stored()))
+
+ litellm_model, provider, credentials = resolve_call(
+ f"{PROVIDER}:{CHAT_MODEL}", kind="chat", config=config
+ )
+
+ assert litellm_model == f"hosted_vllm/{CHAT_MODEL}"
+ assert provider == PROVIDER
+ assert credentials["api_base"] == CHAT_BASE
+
+
+def test_an_embedding_call_resolves_to_the_embedding_endpoint() -> None:
+ """End to end through the gateway: the kind has to survive `resolve_call`,
+ `provider_credentials` and `credential_values` to get here."""
+ config = _config(_providers(**_stored()))
+
+ litellm_model, provider, credentials = resolve_call(
+ f"{PROVIDER}:{EMBED_MODEL}", kind="embedding", config=config
+ )
+
+ assert litellm_model == f"hosted_vllm/{EMBED_MODEL}"
+ assert provider == PROVIDER
+ assert credentials["api_base"] == EMBED_BASE
+
+
+def _config(providers: ProvidersConfig):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_provider=PROVIDER, llm_model=CHAT_MODEL),
+ knowledge=SimpleNamespace(
+ embedding_provider=PROVIDER,
+ embedding_model=EMBED_MODEL,
+ legacy_embedding_provider_map={},
+ ),
+ )
+
+
+def test_credential_values_narrows_to_the_requested_endpoint() -> None:
+ providers = _providers(**_stored())
+
+ assert providers.credential_values(PROVIDER, kind="chat")["api_base"] == CHAT_BASE
+ assert providers.credential_values(PROVIDER, kind="embedding")["api_base"] == EMBED_BASE
+
+
+def test_stored_credentials_keeps_both_endpoints() -> None:
+ """Model discovery has to see both, which the narrowed form cannot give it."""
+ stored = _providers(**_stored()).stored_credentials(PROVIDER)
+
+ assert stored["api_base"] == CHAT_BASE
+ assert stored["embedding_api_base"] == EMBED_BASE
+
+
+def test_the_gateway_asks_the_config_for_the_kind_it_needs() -> None:
+ """`credential_values(provider, kind=...)` is the contract: the gateway
+ passes the keyword outright, so a config object has to declare it."""
+
+ from types import SimpleNamespace
+
+ class _Config:
+ def credential_values(self, provider, *, kind="chat"):
+ return {"api_key": "legacy", "kind": kind}
+
+ credentials = provider_credentials(
+ "openai", SimpleNamespace(providers=_Config()), kind="embedding"
+ )
+
+ assert credentials == {"api_key": "legacy", "kind": "embedding"}
+
+
+def test_a_kindless_enhancement_is_still_callable() -> None:
+ """watsonx.ai on-prem reaches one API and takes the one-argument form."""
+ onprem = registry.get("watsonx_onprem")
+ credentials = registry.credentials_for(
+ onprem,
+ {"api_base": "https://cpd.example.com", "username": "u", "api_key": "k"},
+ "embedding",
+ )
+
+ assert credentials["api_base"] == "https://cpd.example.com"
+
+
+# ---------------------------------------------------------------------------
+# Model discovery
+# ---------------------------------------------------------------------------
+
+
+class _Response:
+ def __init__(self, status_code: int, payload: Any = None, text: str = "") -> None:
+ self.status_code = status_code
+ self._payload = payload
+ self.text = text
+
+ def json(self) -> Any:
+ if self._payload is None:
+ raise ValueError("no JSON body")
+ return self._payload
+
+
+def _models_body(*ids: str) -> dict[str, Any]:
+ return {"object": "list", "data": [{"id": model, "object": "model"} for model in ids]}
+
+
+def _client_returning(responses: dict[str, Any], seen: dict[str, Any] | None = None):
+ """An httpx.AsyncClient stand-in that answers per URL."""
+
+ class _Client:
+ def __init__(self, **kwargs):
+ if seen is not None:
+ seen["verify"] = kwargs.get("verify")
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ async def request(self, method, url, **kwargs):
+ if seen is not None:
+ seen.setdefault("urls", []).append(url)
+ seen["headers"] = kwargs.get("headers")
+ answer = responses[url]
+ if isinstance(answer, Exception):
+ raise answer
+ return answer
+
+ return _Client
+
+
+async def test_each_endpoint_is_asked_what_it_serves(monkeypatch) -> None:
+ """Chat and embedding are told apart by which endpoint answered, so no
+ capability sniffing is needed and the split cannot be wrong."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ },
+ seen,
+ ),
+ )
+
+ models = await rhoai.fetch_models(_stored())
+
+ assert models.chat == (CHAT_MODEL,)
+ assert models.embedding == (EMBED_MODEL,)
+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models"]
+ assert seen["headers"]["Authorization"] == f"Bearer {TOKEN}"
+
+
+async def test_one_endpoint_is_asked_once_and_fills_both_pickers(monkeypatch) -> None:
+ """`GET /v1/models` does not say what a model is for. Emptying a picker on a
+ guess is worse than offering an id that fails loudly if misused."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning({f"{CHAT_BASE}/models": _Response(200, _models_body("solo"))}, seen),
+ )
+
+ models = await rhoai.fetch_models(_stored(embedding_api_base=None))
+
+ assert models.chat == ("solo",)
+ assert models.embedding == ("solo",)
+ assert seen["urls"] == [f"{CHAT_BASE}/models"]
+
+
+async def test_the_configured_tls_setting_is_used_for_discovery_too(monkeypatch) -> None:
+ """Discovery talks to the endpoint directly, not through LiteLLM, so without
+ this it could fail on a certificate the gateway accepts, or the reverse."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ },
+ seen,
+ ),
+ )
+
+ await rhoai.fetch_models(_stored(ssl_verify="false"))
+
+ assert seen["verify"] is False
+
+
+async def test_one_dead_endpoint_does_not_empty_the_other_picker(monkeypatch) -> None:
+ """A rolling embedding deployment must not take the chat picker with it."""
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ models = await rhoai.fetch_models(_stored())
+
+ assert models.chat == (CHAT_MODEL,)
+ assert models.embedding is None # that half keeps its configured fallback
+
+
+async def test_a_dead_endpoint_does_not_empty_its_own_picker_either(monkeypatch) -> None:
+ """The failed half is None, not an empty list, so the catalogue keeps the
+ configured rows for it rather than showing nothing for five minutes."""
+ responses = {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),
+ }
+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses))
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ await rhoai.fetch_models(_stored())
+
+ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)
+
+
+async def test_a_failed_listing_keeps_the_previous_answer_for_that_half(monkeypatch) -> None:
+ """An endpoint mid-rollout answered fine a minute ago; that answer is
+ better than the configured guess, so it stays until it ages out."""
+ seen: dict[str, Any] = {}
+ responses = {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ }
+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses, seen))
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+ await rhoai.fetch_models(_stored())
+
+ # The embedding answer ages out; the endpoint is then down.
+ rhoai._models_cache["embedding"] = rhoai._models_cache["embedding"]._replace(
+ at=time.monotonic() - rhoai.MODELS_TTL_SECONDS - 1
+ )
+ responses[f"{EMBED_BASE}/models"] = _Response(503, None, "Service Unavailable")
+ models = await rhoai.fetch_models(_stored())
+
+ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)
+ # Only the stale half was asked again; the fresh chat answer was reused.
+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models", f"{EMBED_BASE}/models"]
+
+
+async def test_a_failed_half_is_asked_again_next_time(monkeypatch) -> None:
+ """A failure must not be parked behind a fresh timestamp: the next refresh
+ tries that endpoint again, and a recovered answer fills the picker."""
+ seen: dict[str, Any] = {}
+ responses = {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),
+ }
+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses, seen))
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+ await rhoai.fetch_models(_stored())
+
+ responses[f"{EMBED_BASE}/models"] = _Response(200, _models_body(EMBED_MODEL))
+ models = await rhoai.fetch_models(_stored())
+
+ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=(EMBED_MODEL,))
+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models", f"{EMBED_BASE}/models"]
+
+
+async def _no_retry(method, url, *, client, **kwargs):
+ """Skip the backoff so a failure test does not sleep."""
+ return await client.request(method, url, **kwargs)
+
+
+async def test_an_unreachable_cluster_keeps_the_configured_fallback(monkeypatch) -> None:
+ """Returning None leaves whatever the catalogue had; raising would fail the
+ whole catalogue request because one endpoint is down."""
+ import httpx
+
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": httpx.ConnectError("no route to host"),
+ f"{EMBED_BASE}/models": httpx.ConnectError("no route to host"),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ assert await rhoai.fetch_models(_stored()) is None
+
+
+async def test_a_listing_that_cannot_be_read_keeps_the_fallback(monkeypatch) -> None:
+ """A proxy answering with an HTML error page must not put garbage in a picker."""
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, None, "<html>gateway timeout</html>"),
+ f"{EMBED_BASE}/models": _Response(200, {"unexpected": "shape"}),
+ }
+ ),
+ )
+
+ assert await rhoai.fetch_models(_stored()) is None
+
+
+async def test_an_empty_listing_on_one_half_keeps_that_fallback(monkeypatch) -> None:
+ """An endpoint that answers `{"data": []}` while the other half is fine must
+ not have an empty tuple cached for it: that would swap the configured rows
+ for a blank picker until the TTL expires."""
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, {"object": "list", "data": []}),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ models = await rhoai.fetch_models(_stored())
+
+ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)
+ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)
+
+
+async def test_an_unreadable_listing_on_one_half_keeps_that_fallback(monkeypatch) -> None:
+ """Same for a JSON body of the wrong shape: it is no answer for that half,
+ not an answer of "nothing"."""
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, {"unexpected": "shape"}),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ await rhoai.fetch_models(_stored())
+
+ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)
+
+
+async def test_incomplete_credentials_are_not_taken_to_the_network(monkeypatch) -> None:
+ def _explode(**kwargs):
+ raise AssertionError("no HTTP call should be made")
+
+ monkeypatch.setattr("httpx.AsyncClient", _explode)
+
+ assert await rhoai.fetch_models({"api_key": TOKEN}) is None
+ assert await rhoai.fetch_models({"api_base": CHAT_BASE}) is None
+
+
+ "body, expected",
+ [
+ (_models_body("a", "b"), ("a", "b")),
+ # Order preserved, duplicates dropped.
+ (_models_body("a", "b", "a"), ("a", "b")),
+ ({"object": "list", "data": []}, ()),
+ ({"object": "list", "data": [{"no": "id"}, {"id": " "}, "junk"]}, ()),
+ ({"error": {"message": "nope"}}, ()),
+ ("not a mapping", ()),
+ (None, ()),
+ ],
+)
+def test_only_well_formed_model_entries_are_kept(body, expected) -> None:
+ assert rhoai.model_ids(body) == expected
+
+
+async def test_a_fresh_listing_is_reused_rather_than_refetched(monkeypatch) -> None:
+ """The set of served models changes when an administrator deploys one, not
+ once per catalogue request."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ },
+ seen,
+ ),
+ )
+
+ await rhoai.fetch_models(_stored())
+ await rhoai.fetch_models(_stored())
+
+ assert len(seen["urls"]) == 2 # one round, not two
+ assert rhoai.cached_models().chat == (CHAT_MODEL,)
+
+
+async def test_changing_an_endpoint_invalidates_the_cache(monkeypatch) -> None:
+ """A cached list belongs to the endpoints and token it was fetched with."""
+ other = "https://other-chat.svc:8443/v1"
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ f"{other}/models": _Response(200, _models_body("other-model")),
+ },
+ seen,
+ ),
+ )
+
+ await rhoai.fetch_models(_stored())
+ models = await rhoai.fetch_models(_stored(api_base=other))
+
+ assert models.chat == ("other-model",)
+
+
+async def test_forget_models_drops_the_cache(monkeypatch) -> None:
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ }
+ ),
+ )
+
+ await rhoai.fetch_models(_stored())
+ assert rhoai.cached_models() is not None
+
+ rhoai.forget_models()
+ assert rhoai.cached_models() is None
+
+
+# ---------------------------------------------------------------------------
+# Health
+# ---------------------------------------------------------------------------
+
+
+async def test_a_healthy_deployment_passes_with_no_model_selected(monkeypatch) -> None:
+ """`GET /v1/models` needs no model and bills no tokens, which is what makes
+ it a check the provider can pass during onboarding."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),
+ },
+ seen,
+ ),
+ )
+
+ await rhoai.lightweight_health_check(_stored())
+
+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models"]
+
+
+async def test_the_litellm_form_still_checks_with_the_configured_tls_setting(monkeypatch) -> None:
+ """Handed the narrowed bag, the check covers the one endpoint in it — but
+ with the TLS setting the operator chose, not verification switched back on."""
+ seen: dict[str, Any] = {}
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning({f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL))}, seen),
+ )
+ translated = rhoai.litellm_credentials(_stored(ssl_verify="false"), kind="chat")
+
+ await rhoai.lightweight_health_check(translated)
+
+ assert seen["verify"] is False
+ assert seen["urls"] == [f"{CHAT_BASE}/models"]
+
+
+async def test_validation_hands_the_check_the_stored_form(monkeypatch) -> None:
+ """`validate_provider_setup` gets the LiteLLM form for the probe and the
+ stored form for the enhancement check; the check must get the latter, or
+ the embedding endpoint is never looked at before the first ingest."""
+ from api import provider_validation
+
+ received: list[dict[str, Any]] = []
+
+ async def _capture(credentials):
+ received.append(dict(credentials))
+
+ monkeypatch.setattr(rhoai, "lightweight_health_check", _capture)
+ stored = _stored()
+ translated = rhoai.litellm_credentials(stored, kind="chat")
+
+ await provider_validation.validate_provider_setup(
+ provider=PROVIDER, credentials=translated, stored_credentials=stored
+ )
+ await provider_validation.validate_provider_setup(provider=PROVIDER, credentials=translated)
+
+ assert received[0]["embedding_api_base"] == EMBED_BASE
+ assert "embedding_api_base" not in received[1] # no stored form given: falls back
+
+
+async def test_a_wrong_embedding_url_is_found_before_the_first_ingest(monkeypatch) -> None:
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(404, {"error": {"message": "Not Found"}}),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ with pytest.raises(Exception, match="no /v1/models"):
+ await rhoai.lightweight_health_check(_stored())
+
+
[email protected]("status", [401, 403])
+async def test_a_rejected_token_names_the_rolebinding(monkeypatch, status) -> None:
+ """RHOAI runs a SubjectAccessReview, so the usual cause is a missing grant
+ rather than a bad token — and the message has to say so or the operator
+ hunts for the wrong thing."""
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {f"{CHAT_BASE}/models": _Response(status, {"error": {"message": "forbidden"}})}
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ with pytest.raises(Exception, match="inferenceservices"):
+ await rhoai.lightweight_health_check(_stored())
+
+
+async def test_an_unreachable_endpoint_says_which_one(monkeypatch) -> None:
+ import httpx
+
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning({f"{CHAT_BASE}/models": httpx.ConnectError("no route to host")}),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ with pytest.raises(Exception, match="chat endpoint"):
+ await rhoai.lightweight_health_check(_stored())
+
+
+async def test_a_slow_endpoint_is_reported_as_a_timeout(monkeypatch) -> None:
+ import httpx
+
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning({f"{CHAT_BASE}/models": httpx.ConnectTimeout("too slow")}),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+ with pytest.raises(Exception, match="did not respond in time"):
+ await rhoai.lightweight_health_check(_stored())
+
+
+async def test_the_whole_check_is_bounded_however_slow_the_endpoints_are(monkeypatch) -> None:
+ """Per-request timeouts, retries and backoff across two endpoints add up to
+ about a minute; a human is waiting on the settings save, so the check as a
+ whole has a deadline of its own."""
+ import asyncio
+
+ class _Hanging:
+ def __init__(self, **kwargs):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ async def request(self, method, url, **kwargs):
+ await asyncio.sleep(60)
+
+ monkeypatch.setattr("httpx.AsyncClient", _Hanging)
+ monkeypatch.setattr(rhoai, "HEALTH_TOTAL_BUDGET_SECONDS", 0.05)
+
+ with pytest.raises(Exception, match="did not respond in time"):
+ await asyncio.wait_for(rhoai.lightweight_health_check(_stored()), timeout=5)
+
+
+async def test_the_health_check_retries_less_than_discovery(monkeypatch) -> None:
+ """Discovery can afford to wait out a rollout; the pre-save check cannot."""
+ calls: list[int] = []
+
+ async def _count(method, url, *, client, max_retries=2, **kwargs):
+ calls.append(max_retries)
+ return _Response(200, _models_body(CHAT_MODEL))
+
+ monkeypatch.setattr("httpx.AsyncClient", _client_returning({}))
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _count)
+
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+ assert calls == [rhoai.HEALTH_MAX_RETRIES]
+ assert rhoai.HEALTH_MAX_RETRIES < 2
+
+
+async def test_missing_credentials_say_what_is_missing(monkeypatch) -> None:
+ """Distinct messages, because the fix is different: one is a URL, the other
+ is an `oc create token`."""
+
+ def _explode(**kwargs):
+ raise AssertionError("no HTTP call should be made")
+
+ monkeypatch.setattr("httpx.AsyncClient", _explode)
+
+ with pytest.raises(Exception, match="No chat endpoint"):
+ await rhoai.lightweight_health_check({"api_key": TOKEN})
+
+ with pytest.raises(Exception, match="No ServiceAccount token"):
+ await rhoai.lightweight_health_check({"api_base": CHAT_BASE})
+
+
+# ---------------------------------------------------------------------------
+# Visibility and the catalogue
+# ---------------------------------------------------------------------------
+
+
[email protected]("run_mode", ["oss", "on_prem", "saas"])
+def test_it_is_hidden_everywhere_as_shipped(monkeypatch, run_mode) -> None:
+ """Its endpoints and token are deployment-specific, so it is turned on with
+ OPENRAG_MODEL_PROVIDERS_CONFIG rather than offered to everyone."""
+ monkeypatch.delenv("OPENRAG_MODEL_PROVIDERS_CONFIG", raising=False)
+ monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)
+ model_providers.reload()
+
+ assert PROVIDER not in model_providers.visible_provider_keys()
+
+
+def test_a_deployment_can_offer_it_with_a_config_override(tmp_path, monkeypatch) -> None:
+ _enable_in_oss(tmp_path, monkeypatch)
+
+ assert PROVIDER in model_providers.visible_provider_keys()
+
+
+def test_the_configured_models_are_the_fallback_picker(tmp_path, monkeypatch) -> None:
+ """LiteLLM's bundled table has no hosted_vllm models at all, so without the
+ config rows the pickers would be empty until the cluster answers."""
+ _enable_in_oss(tmp_path, monkeypatch)
+
+ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]
+
+ assert [m["model"] for m in entry["models"]] == [CHAT_MODEL]
+ assert [m["model"] for m in entry["embedding_models"]] == [EMBED_MODEL]
+ assert entry["name"] == "Red Hat OpenShift AI"
+
+
+async def test_what_the_cluster_serves_wins_over_the_configured_fallback(
+ tmp_path, monkeypatch
+) -> None:
+ """A model the InferenceServices do not serve must not sit in the picker
+ waiting to be chosen."""
+ _enable_in_oss(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body("deployed-chat")),
+ f"{EMBED_BASE}/models": _Response(200, _models_body("deployed-embed")),
+ }
+ ),
+ )
+ await rhoai.fetch_models(_stored())
+
+ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]
+
+ assert [m["model"] for m in entry["models"]] == ["deployed-chat"]
+ assert [m["model"] for m in entry["embedding_models"]] == ["deployed-embed"]
+
+
+async def test_the_picker_for_a_dead_endpoint_keeps_the_configured_fallback(
+ tmp_path, monkeypatch
+) -> None:
+ """One endpoint down empties neither picker: the live answer wins where
+ there is one, and the configured rows stand in for the half without."""
+ _enable_in_oss(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ "httpx.AsyncClient",
+ _client_returning(
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body("deployed-chat")),
+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),
+ }
+ ),
+ )
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+ await rhoai.fetch_models(_stored())
+
+ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]
+
+ assert [m["model"] for m in entry["models"]] == ["deployed-chat"]
+ assert [m["model"] for m in entry["embedding_models"]] == [EMBED_MODEL]
+
+
+async def test_the_catalogue_refresh_passes_both_endpoints(tmp_path, monkeypatch) -> None:
+ """`refresh_live_models` reads the stored form, not the narrowed LiteLLM one:
+ the kind-specific bag carries only one endpoint, and discovery needs both."""
+ _enable_in_oss(tmp_path, monkeypatch)
+ captured: dict[str, Any] = {}
+
+ async def _capture(credentials):
+ captured.update(credentials)
+ return None
+
+ monkeypatch.setattr(rhoai, "fetch_models", _capture)
+ monkeypatch.setattr(
+ "config.settings.get_openrag_config", lambda: _config(_providers(**_stored()))
+ )
+
+ await model_catalog.refresh_live_models()
+
+ assert captured["api_base"] == CHAT_BASE
+ assert captured["embedding_api_base"] == EMBED_BASE
diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.py
index d475ce642..2f872ee4d 100644
--- a/tests/unit/test_provider_error_formatting.py
+++ b/tests/unit/test_provider_error_formatting.py
@@ -241,7 +241,7 @@ async def test_resolve_ingest_error_message_probes_generic_embedding_with_creden
watsonx = None
ollama = None
- def credential_values(self, provider):
+ def credential_values(self, provider, *, kind="chat"):
assert provider == "azure"
return dict(credentials)
@@ -281,6 +281,84 @@ async def test_resolve_ingest_error_message_probes_generic_embedding_with_creden
assert len(calls) == 1
+async def test_ingest_recovery_probes_request_embedding_credentials(monkeypatch):
+ """The embedding probes must ask for the *embedding* endpoint's credentials.
+
+ A provider that serves chat and embeddings from separate endpoints (Red Hat
+ OpenShift AI) narrows ``credential_values`` by ``kind``; probing the
+ embedding model against the chat endpoint would report a bogus "model not
+ found" and hide the real embedding-endpoint failure.
+ """
+
+ by_kind = {
+ "chat": {"api_key": "token", "api_base": "https://chat.example/v1"},
+ "embedding": {"api_key": "token", "api_base": "https://embed.example/v1"},
+ }
+
+ class GenericProvider:
+ configured = True
+
+ class FakeProviders:
+ custom = {"rhoai": GenericProvider()}
+ openai = None
+ anthropic = None
+ watsonx = None
+ ollama = None
+
+ def credential_values(self, provider, *, kind="chat"):
+ assert provider == "rhoai"
+ return dict(by_kind[kind])
+
+ class FakeConfig:
+ class knowledge:
+ embedding_provider = "rhoai"
+ embedding_model = "granite-embedding"
+
+ class agent:
+ llm_provider = "rhoai"
+ llm_model = "granite-chat"
+
+ providers = FakeProviders()
+
+ def get_embedding_provider_config(self):
+ return self.providers.custom["rhoai"]
+
+ def get_llm_provider_config(self):
+ return self.providers.custom["rhoai"]
+
+ calls = []
+
+ async def fake_validate(**kwargs):
+ calls.append(kwargs)
+
+ monkeypatch.setattr("config.settings.get_openrag_config", lambda: FakeConfig())
+ monkeypatch.setattr("api.provider_validation.validate_provider_setup", fake_validate)
+
+ from api.provider_validation import (
+ probe_chat_llm_error,
+ probe_embedding_error,
+ probe_provider_credential_error,
+ )
+
+ assert await probe_embedding_error() is None
+ assert await probe_chat_llm_error() is None
+ assert await probe_provider_credential_error() is None
+
+ # probe_embedding_error, probe_chat_llm_error, then the credential probe's
+ # single rhoai candidate (the embedding row is checked first, so it wins).
+ assert [c["credentials"]["api_base"] for c in calls] == [
+ "https://embed.example/v1",
+ "https://chat.example/v1",
+ "https://embed.example/v1",
+ ]
+ assert [c["endpoint"] for c in calls] == [
+ "https://embed.example/v1",
+ "https://chat.example/v1",
+ "https://embed.example/v1",
+ ]
+
+
@pytest.mark.asyncio
async def test_probe_chat_llm_error_uses_generic_provider_credentials(monkeypatch):
credentials = {
@@ -295,7 +373,7 @@ async def test_probe_chat_llm_error_uses_generic_provider_credentials(monkeypatc
class FakeProviders:
custom = {"azure": GenericProvider()}
- def credential_values(self, provider):
+ def credential_values(self, provider, *, kind="chat"):
assert provider == "azure"
return dict(credentials)
diff --git a/tests/unit/test_provider_health_generic.py b/tests/unit/test_provider_health_generic.py
index b6c064481..e18c1c656 100644
--- a/tests/unit/test_provider_health_generic.py
+++ b/tests/unit/test_provider_health_generic.py
@@ -20,13 +20,15 @@ async def test_provider_health_accepts_configured_azure_provider(
endpoint=None,
project_id=None,
)
+ stored = {
+ "api_key": "azure-secret",
+ "api_base": "https://example.openai.azure.com",
+ "api_version": "2024-10-21",
+ }
providers = SimpleNamespace(
get_provider_config=lambda provider: azure,
- credential_values=lambda provider: {
- "api_key": "azure-secret",
- "api_base": "https://example.openai.azure.com",
- "api_version": "2024-10-21",
- },
+ credential_values=lambda provider, kind="chat": dict(stored),
+ stored_credentials=lambda provider: dict(stored),
)
config = SimpleNamespace(
providers=providers,
@@ -57,14 +59,112 @@ async def test_provider_health_accepts_configured_azure_provider(
endpoint=None,
project_id=None,
test_completion=True,
- credentials={
- "api_key": "azure-secret",
- "api_base": "https://example.openai.azure.com",
- "api_version": "2024-10-21",
- },
+ credentials=stored,
+ stored_credentials=stored,
)
+async def test_provider_health_forwards_the_stored_form_for_multi_endpoint_providers(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """OpenShift AI keeps two endpoints; `credential_values` narrows to one of
+ them for the LiteLLM probe, so the enhancement's health check is handed the
+ stored form separately, or the embedding endpoint is never checked."""
+ stored = {
+ "api_base": "https://chat.svc:8443/v1",
+ "embedding_api_base": "https://embed.svc:8443/v1",
+ "api_key": "sha256~token",
+ "ssl_verify": "false",
+ }
+ translated = {"api_key": "sha256~token", "api_base": stored["api_base"], "ssl_verify": False}
+ providers = SimpleNamespace(
+ get_provider_config=lambda provider: SimpleNamespace(
+ api_key=None, endpoint=None, project_id=None
+ ),
+ credential_values=lambda provider, kind="chat": dict(translated),
+ stored_credentials=lambda provider: dict(stored),
+ )
+ config = SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_provider="openai", llm_model="gpt-4o-mini"),
+ knowledge=SimpleNamespace(embedding_provider="openai", embedding_model="ada"),
+ )
+ validate = AsyncMock()
+ monkeypatch.setattr("api.provider_health.get_openrag_config", lambda: config)
+ monkeypatch.setattr("api.provider_health.validate_provider_setup", validate)
+ monkeypatch.setattr("api.provider_health.is_known_provider", lambda provider: True)
+
+ response = await check_provider_health(provider="rhoai", user=None)
+
+ assert response.status_code == 200
+ # Not selected for either role, so there is no model to probe with: one
+ # lightweight check, handed both forms.
+ validate.assert_awaited_once()
+ kwargs = validate.await_args.kwargs
+ assert kwargs["llm_model"] is None
+ assert kwargs["embedding_model"] is None
+ assert kwargs["credentials"] == translated
+ assert kwargs["stored_credentials"] == stored
+
+
+async def test_provider_health_probes_each_role_of_a_dual_role_provider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A provider selected for both roles is probed once per role, each with
+ that role's credentials. The validator tests one model per call, so a single
+ call would probe the embedding model only and report the chat model as
+ validated without ever touching its endpoint."""
+ by_kind = {
+ "chat": {"api_key": "sha256~token", "api_base": "https://chat.svc:8443/v1"},
+ "embedding": {"api_key": "sha256~token", "api_base": "https://embed.svc:8443/v1"},
+ }
+ stored = {
+ "api_key": "sha256~token",
+ "api_base": by_kind["chat"]["api_base"],
+ "embedding_api_base": by_kind["embedding"]["api_base"],
+ }
+ providers = SimpleNamespace(
+ get_provider_config=lambda provider: SimpleNamespace(
+ api_key=None, endpoint=None, project_id=None
+ ),
+ credential_values=lambda provider, kind="chat": dict(by_kind[kind]),
+ stored_credentials=lambda provider: dict(stored),
+ )
+ config = SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="granite-3.3-2b-instruct"),
+ knowledge=SimpleNamespace(
+ embedding_provider="rhoai", embedding_model="granite-embedding-english-r2"
+ ),
+ )
+ validate = AsyncMock()
+ monkeypatch.setattr("api.provider_health.get_openrag_config", lambda: config)
+ monkeypatch.setattr("api.provider_health.validate_provider_setup", validate)
+ monkeypatch.setattr("api.provider_health.is_known_provider", lambda provider: True)
+
+ response = await check_provider_health(provider="rhoai", user=None)
+
+ assert response.status_code == 200
+ body = json.loads(response.body)
+ assert body["details"] == {
+ "llm_model": "granite-3.3-2b-instruct",
+ "embedding_model": "granite-embedding-english-r2",
+ "endpoint": None,
+ }
+ assert validate.await_count == 2
+ chat, embedding = (call.kwargs for call in validate.await_args_list)
+ assert (chat["llm_model"], chat["embedding_model"]) == ("granite-3.3-2b-instruct", None)
+ assert chat["credentials"] == by_kind["chat"]
+ assert (embedding["llm_model"], embedding["embedding_model"]) == (
+ None,
+ "granite-embedding-english-r2",
+ )
+ assert embedding["credentials"] == by_kind["embedding"]
+ assert all(call.kwargs["stored_credentials"] == stored for call in validate.await_args_list)
+
+
@pytest.mark.asyncio
async def test_provider_page_catalog_contract_includes_azure(
monkeypatch: pytest.MonkeyPatch,