OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Models API should return sanitized provider error messages."""
from __future__ import annotations
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from fastapi.responses import JSONResponse
from api import models as models_api
@pytest.mark.asyncio
async def test_get_ibm_models_returns_sanitized_provider_error():
raw = (
'{"errorCode":"BXNIM0415E",'
'"errorMessage":"Provided API key could not be found.",'
'"context":{"requestId":"abc"}}'
)
models_service = SimpleNamespace(
get_ibm_models=AsyncMock(side_effect=Exception(raw)),
)
response = await models_api.get_ibm_models(
body=models_api.IBMBody(
api_key="bad-key",
endpoint="https://ca-tor.ml.cloud.ibm.com",
project_id="proj",
),
models_service=models_service,
user=SimpleNamespace(),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
payload = json.loads(response.body)
assert payload["error"] == "Provided API key is Invalid."
@pytest.mark.asyncio
async def test_get_openai_models_returns_sanitized_provider_error():
raw = json.dumps(
{
"error": {
"message": "Incorrect API key provided: sk-bad.",
"type": "invalid_request_error",
"code": "invalid_api_key",
}
}
)
models_service = SimpleNamespace(
get_openai_models=AsyncMock(side_effect=Exception(raw)),
)
response = await models_api.get_openai_models(
body=models_api.OpenAIBody(api_key="sk-bad"),
models_service=models_service,
user=SimpleNamespace(),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
payload = json.loads(response.body)
assert payload["error"] == "Incorrect API key provided: [REDACTED]."
assert "sk-bad" not in payload["error"]
@pytest.mark.asyncio
async def test_get_openai_models_returns_502_on_transport_error():
import httpx
models_service = SimpleNamespace(
get_openai_models=AsyncMock(
side_effect=httpx.ConnectError("All connection attempts failed")
),
)
response = await models_api.get_openai_models(
body=models_api.OpenAIBody(api_key="sk-test"),
models_service=models_service,
user=SimpleNamespace(),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 502
payload = json.loads(response.body)
assert payload["error"] == "Unable to reach the model provider. Please try again."
assert "connection attempts" not in payload["error"].lower()
@pytest.mark.asyncio
async def test_get_openai_models_returns_500_on_unexpected_error():
models_service = SimpleNamespace(
get_openai_models=AsyncMock(side_effect=RuntimeError("boom secret-key")),
)
response = await models_api.get_openai_models(
body=models_api.OpenAIBody(api_key="sk-test"),
models_service=models_service,
user=SimpleNamespace(),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 500
payload = json.loads(response.body)
assert payload["error"] == "An unexpected error occurred while fetching models."
assert "boom" not in payload["error"]
@pytest.mark.asyncio
async def test_get_ibm_models_returns_project_configuration_error():
"""Bare Exception from models_service is the user-facing contract — do not 500."""
msg = (
"API key is valid, but no models are available. "
"This usually means your Watson Machine Learning (WML) project is not properly configured."
)
models_service = SimpleNamespace(
get_ibm_models=AsyncMock(side_effect=Exception(msg)),
)
response = await models_api.get_ibm_models(
body=models_api.IBMBody(
api_key="ok-key",
endpoint="https://ca-tor.ml.cloud.ibm.com",
project_id="proj",
),
models_service=models_service,
user=SimpleNamespace(),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
payload = json.loads(response.body)
assert payload["error"] == msg
@pytest.mark.asyncio
async def test_watsonx_space_listing_merges_pending_fields_with_saved_secret(monkeypatch):
from config.config_manager import (
AnthropicConfig,
GenericProviderConfig,
OllamaConfig,
OpenAIConfig,
ProvidersConfig,
WatsonXConfig,
)
from enhancements.providers.watsonx import onprem
providers = ProvidersConfig(
openai=OpenAIConfig(),
anthropic=AnthropicConfig(),
watsonx=WatsonXConfig(),
ollama=OllamaConfig(),
custom={
onprem.PROVIDER_KEY: GenericProviderConfig(
credentials={
"api_base": "https://cpd.example.com",
"username": "saved-user",
"api_key": "saved-secret",
"ssl_verify": "true",
},
auth_method="username_api_key",
configured=True,
)
},
)
monkeypatch.setattr(
models_api,
"get_openrag_config",
lambda: SimpleNamespace(providers=providers),
)
seen = {}
async def _list_spaces(credentials):
seen.update(credentials)
return [{"id": "space-1", "name": "Production"}]
monkeypatch.setattr(onprem, "list_spaces", _list_spaces)
response = await models_api.get_watsonx_onprem_spaces(
body=models_api.WatsonxOnPremSpacesBody(
credentials={"username": "pending-user", "api_key": ""},
auth_method="username_api_key",
),
user=SimpleNamespace(),
)
assert response.status_code == 200
assert json.loads(response.body) == {"spaces": [{"id": "space-1", "name": "Production"}]}
assert seen == {
"api_base": "https://cpd.example.com",
"username": "pending-user",
"api_key": "saved-secret",
"ssl_verify": "true",
}
@pytest.mark.asyncio
async def test_watsonx_space_listing_never_reuses_secrets_for_a_new_target(monkeypatch):
from config.config_manager import (
AnthropicConfig,
GenericProviderConfig,
OllamaConfig,
OpenAIConfig,
ProvidersConfig,
WatsonXConfig,
)
from enhancements.providers.watsonx import onprem
providers = ProvidersConfig(
openai=OpenAIConfig(),
anthropic=AnthropicConfig(),
watsonx=WatsonXConfig(),
ollama=OllamaConfig(),
custom={
onprem.PROVIDER_KEY: GenericProviderConfig(
credentials={
"api_base": "https://saved.example.com",
"username": "saved-user",
"api_key": "saved-secret",
"ssl_verify": "true",
},
auth_method="username_api_key",
configured=True,
)
},
)
monkeypatch.setattr(
models_api,
"get_openrag_config",
lambda: SimpleNamespace(providers=providers),
)
seen = {}
async def _list_spaces(credentials):
seen.update(credentials)
return []
monkeypatch.setattr(onprem, "list_spaces", _list_spaces)
response = await models_api.get_watsonx_onprem_spaces(
body=models_api.WatsonxOnPremSpacesBody(
credentials={"api_base": "https://new.example.com"},
auth_method="username_api_key",
),
user=SimpleNamespace(),
)
assert response.status_code == 200
assert seen == {"api_base": "https://new.example.com"}
@pytest.mark.asyncio
async def test_watsonx_space_listing_hides_upstream_error_details(monkeypatch):
from config.config_manager import (
AnthropicConfig,
GenericProviderConfig,
OllamaConfig,
OpenAIConfig,
ProvidersConfig,
WatsonXConfig,
)
from enhancements.providers.watsonx import onprem
providers = ProvidersConfig(
openai=OpenAIConfig(),
anthropic=AnthropicConfig(),
watsonx=WatsonXConfig(),
ollama=OllamaConfig(),
custom={
onprem.PROVIDER_KEY: GenericProviderConfig(
credentials={
"api_base": "https://cpd.example.com",
"username": "saved-user",
"api_key": "saved-secret",
},
auth_method="username_api_key",
configured=True,
)
},
)
monkeypatch.setattr(
models_api,
"get_openrag_config",
lambda: SimpleNamespace(providers=providers),
)
async def _list_spaces(credentials):
raise onprem.SpaceDiscoveryError(
401,
"Traceback: Authorization Bearer exposed-secret",
)
monkeypatch.setattr(onprem, "list_spaces", _list_spaces)
response = await models_api.get_watsonx_onprem_spaces(
body=models_api.WatsonxOnPremSpacesBody(),
user=SimpleNamespace(),
)
assert response.status_code == 401
assert json.loads(response.body) == {
"error": "The cluster rejected the configured credentials."
}
assert b"exposed-secret" not in response.body
@pytest.mark.asyncio
async def test_watsonx_space_listing_infers_the_auth_method_from_a_saved_zen_key(monkeypatch):
from config.config_manager import (
AnthropicConfig,
GenericProviderConfig,
OllamaConfig,
OpenAIConfig,
ProvidersConfig,
WatsonXConfig,
)
from enhancements.providers.watsonx import onprem
providers = ProvidersConfig(
openai=OpenAIConfig(),
anthropic=AnthropicConfig(),
watsonx=WatsonXConfig(),
ollama=OllamaConfig(),
custom={
onprem.PROVIDER_KEY: GenericProviderConfig(
credentials={
"api_base": "https://cpd.example.com",
"zen_api_key": "saved-zen",
},
configured=True,
)
},
)
monkeypatch.setattr(
models_api,
"get_openrag_config",
lambda: SimpleNamespace(providers=providers),
)
seen = {}
async def _list_spaces(credentials):
seen.update(credentials)
return []
monkeypatch.setattr(onprem, "list_spaces", _list_spaces)
response = await models_api.get_watsonx_onprem_spaces(
body=models_api.WatsonxOnPremSpacesBody(credentials={"username": "typed-user"}),
user=SimpleNamespace(),
)
assert response.status_code == 200
# Zen fields apply, so the username typed for the other method is dropped.
assert seen == {"api_base": "https://cpd.example.com", "zen_api_key": "saved-zen"}
@pytest.mark.asyncio
async def test_watsonx_space_listing_rejects_an_unknown_auth_method(monkeypatch):
from config.config_manager import (
AnthropicConfig,
OllamaConfig,
OpenAIConfig,
ProvidersConfig,
WatsonXConfig,
)
from enhancements.providers.watsonx import onprem
providers = ProvidersConfig(
openai=OpenAIConfig(),
anthropic=AnthropicConfig(),
watsonx=WatsonXConfig(),
ollama=OllamaConfig(),
)
monkeypatch.setattr(
models_api,
"get_openrag_config",
lambda: SimpleNamespace(providers=providers),
)
list_spaces = AsyncMock(return_value=[])
monkeypatch.setattr(onprem, "list_spaces", list_spaces)
response = await models_api.get_watsonx_onprem_spaces(
body=models_api.WatsonxOnPremSpacesBody(
credentials={"api_base": "https://cpd.example.com"},
auth_method="bogus",
),
user=SimpleNamespace(),
)
assert response.status_code == 400
assert json.loads(response.body) == {"error": models_api._INVALID_AUTH_METHOD_ERROR}
list_spaces.assert_not_awaited()
@pytest.mark.asyncio
async def test_get_model_catalog_returns_only_supported_providers():
"""The catalogue publishes exactly what this run mode's config exposes.
Not a fixed four: `config/model_providers.yaml` decides, so pinning a
literal set here would only re-hardcode the list the config replaced.
"""
from services.model_catalog import supported_provider_keys
response = await models_api.get_model_catalog(user=SimpleNamespace())
assert isinstance(response, JSONResponse)
assert response.status_code == 200
payload = json.loads(response.body)
keys = {provider["key"] for provider in payload["providers"]}
assert keys == supported_provider_keys()
openai = next(p for p in payload["providers"] if p["key"] == "openai")
assert openai["models"]
assert openai["embedding_models"]
cache_control = response.headers.get("cache-control") or response.headers.get("Cache-Control")
# Never `max-age`: a cached copy outlives the restart that applies a
# config edit, so the picker keeps showing the previous model list.
assert cache_control == "no-store"
@pytest.mark.asyncio
async def test_get_model_catalog_hides_catalog_exception_text(monkeypatch):
"""CodeQL py/stack-trace-exposure: log the cause, return a fixed message."""
from services import model_catalog
def boom(*args, **kwargs):
raise model_catalog.CatalogUnavailableError("litellm is not installed on the server")
monkeypatch.setattr(model_catalog, "catalog", boom)
response = await models_api.get_model_catalog(user=SimpleNamespace())
assert response.status_code == 503
body = response.body.decode()
assert json.loads(body)["error"] == model_catalog.CATALOG_UNAVAILABLE_MESSAGE
assert "litellm" not in body