OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""`validate_provider_setup` says what a passing validation actually exercised.
The health check erases a real-traffic failure only on evidence as strong as
that failure, so each branch has to report honestly: a deployment listing, a
key check, or a missing model is not a model call, and the LiteLLM probe's
plain completion is not a tool-calling one.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from api import provider_validation
from api.provider_validation import ProbeResult, validate_provider_setup
@pytest.fixture
def calls(monkeypatch):
"""Stub every probe the validator can reach, recording which one ran."""
ran: list[str] = []
def stub(name):
async def probe(*_args, **_kwargs):
ran.append(name)
return probe
for name in (
"_test_litellm_provider",
"test_lightweight_health",
"test_embedding",
"test_completion_with_tools",
):
monkeypatch.setattr(provider_validation, name, stub(name))
monkeypatch.setattr(provider_validation, "get_provider_enhancement", lambda _p: None)
return ran
@pytest.mark.asyncio
async def test_a_native_tool_calling_completion_is_reported_as_one(calls):
result = await validate_provider_setup(
provider="openai", api_key="k", llm_model="gpt-4o", test_completion=True
)
assert calls == ["test_completion_with_tools"]
assert result == ProbeResult(model_probed=True, tools_exercised=True)
@pytest.mark.asyncio
async def test_a_native_embedding_call_probes_the_model(calls):
result = await validate_provider_setup(
provider="openai",
api_key="k",
embedding_model="text-embedding-3-small",
test_completion=True,
)
assert calls == ["test_embedding"]
assert result == ProbeResult(model_probed=True)
@pytest.mark.asyncio
async def test_the_litellm_probe_calls_the_model_without_tools(monkeypatch, calls):
"""RHOAI and every other enhancement: a plain `Reply with OK.` completion."""
monkeypatch.setattr(
provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()
)
monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})
result = await validate_provider_setup(
provider="rhoai", llm_model="granite", test_completion=True, credentials={}
)
assert calls == ["_test_litellm_provider"]
assert result == ProbeResult(model_probed=True, tools_exercised=False)
@pytest.fixture
def enhancement(monkeypatch):
monkeypatch.setattr(
provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()
)
monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})
@pytest.mark.asyncio
@pytest.mark.parametrize("model", [{"llm_model": "granite"}, {"embedding_model": "granite-emb"}])
async def test_a_polled_enhancement_check_calls_no_model(calls, enhancement, model):
"""The 30s health poll must not run inference on RHOAI / watsonx.ai on-prem.
It held a GPU slot (or billed) on every poll, and its pooled connections
raced vLLM's 5s keep-alive into "Server disconnected" banners.
"""
result = await validate_provider_setup(provider="rhoai", credentials={}, **model)
assert calls == ["test_lightweight_health"]
assert result == ProbeResult()
@pytest.mark.asyncio
async def test_a_save_still_probes_the_enhancement_model(calls, enhancement):
"""`verify_model` keeps the pre-save check that rejects an unserved model."""
result = await validate_provider_setup(
provider="rhoai", llm_model="granite", credentials={}, verify_model=True
)
assert calls == ["_test_litellm_provider"]
assert result == ProbeResult(model_probed=True, tools_exercised=False)
@pytest.mark.asyncio
async def test_verify_model_does_not_change_a_native_provider(calls):
"""Native providers keep their own lightweight check on save."""
result = await validate_provider_setup(
provider="openai", api_key="k", llm_model="gpt-4o", verify_model=True
)
assert calls == ["test_lightweight_health"]
assert result == ProbeResult()
@pytest.mark.asyncio
async def test_azure_lists_deployments_and_calls_no_model(calls):
result = await validate_provider_setup(
provider="azure",
api_key="k",
endpoint="https://example.openai.azure.com",
llm_model="my-deployment",
test_completion=True,
)
assert calls == ["test_lightweight_health"]
assert result == ProbeResult()
@pytest.mark.asyncio
async def test_no_model_means_no_call(calls):
"""Azure after #2384 leaves the model empty; nothing is probed then."""
result = await validate_provider_setup(provider="openai", api_key="k", test_completion=True)
assert calls == []
assert result == ProbeResult()
@pytest.mark.asyncio
async def test_a_lightweight_check_calls_no_model(calls):
result = await validate_provider_setup(provider="openai", api_key="k", llm_model="gpt-4o")
assert calls == ["test_lightweight_health"]
assert result == ProbeResult()