OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""The health banner has to be able to clear itself.
`provider_error_log` is erased by the next call to a provider that succeeds,
and until this was wired up the only such call was chat traffic. One failed
turn latched the banner, the frontend then polled every 5s with
`test_completion` while it stayed latched, and a provider that was
demonstrably serving kept being reported broken — offering "Fix Setup" for a
setup that had just passed its own check — until the entry went stale a
quarter of an hour later.
It must not clear itself on weaker evidence than the failure it erases,
though. Only a probe that sent the same shape of request — a tool-calling
completion for chat, a real embedding call for embeddings — may do it.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from api import provider_health
from api.provider_validation import ProbeResult
from services import provider_error_log
#: What each kind of passing validation actually exercised.
TOOL_COMPLETION = ProbeResult(model_probed=True, tools_exercised=True)
PLAIN_COMPLETION = ProbeResult(model_probed=True)
NO_CALL = ProbeResult()
@pytest.fixture(autouse=True)
def _clean():
provider_error_log.clear()
yield
provider_error_log.clear()
def _config():
providers = SimpleNamespace(
credential_values=lambda *_a, **_k: {},
stored_credentials=lambda *_a, **_k: {},
get_provider_config=lambda *_a, **_k: SimpleNamespace(api_key="k"),
)
return SimpleNamespace(
providers=providers,
agent=SimpleNamespace(llm_provider="rhoai", llm_model="gpt-oss-120b"),
knowledge=SimpleNamespace(embedding_provider="rhoai", embedding_model="granite-embedding"),
get_llm_provider_config=lambda: SimpleNamespace(api_key="k"),
get_embedding_provider_config=lambda: SimpleNamespace(api_key="k"),
)
def _probes(monkeypatch, *, chat: ProbeResult, embedding: ProbeResult) -> None:
"""Validation passes for both roles, having exercised what is given."""
async def ok(**kwargs):
return embedding if kwargs.get("embedding_model") else chat
monkeypatch.setattr(provider_health, "validate_provider_setup", ok)
@pytest.fixture
def _healthy_probe(monkeypatch):
"""A provider whose probes all pass, with the caches out of the way."""
monkeypatch.setattr(provider_health, "get_openrag_config", _config)
monkeypatch.setattr(provider_health, "is_known_provider", lambda _p: True)
_probes(monkeypatch, chat=TOOL_COMPLETION, embedding=PLAIN_COMPLETION)
monkeypatch.setattr(provider_health.provider_health_cache, "cache_key", lambda **_k: "key")
monkeypatch.setattr(provider_health.provider_health_cache, "get", lambda _k: None)
monkeypatch.setattr(
provider_health.provider_health_cache, "set_and_release", lambda *_a, **_k: None
)
monkeypatch.setattr(provider_health.provider_health_cache, "release_error", lambda *_a: None)
async def acquire(_key):
return True
monkeypatch.setattr(provider_health.provider_health_cache, "acquire", acquire)
# Keep the corpus diagnostics out of it: unstubbed, they read the real
# config, can list models on a real cluster, and query OpenSearch.
async def _no_refresh(_provider):
return None
async def _no_stale(_provider):
return None
monkeypatch.setattr(provider_health, "_refresh_live_models", _no_refresh)
monkeypatch.setattr(provider_health, "_stale_embedding_spaces", _no_stale)
@pytest.mark.asyncio
async def test_a_passing_tool_calling_probe_clears_a_latched_chat_failure(_healthy_probe):
provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
response = await provider_health.check_provider_health(test_completion=True, user=None)
assert response.status_code == 200
assert provider_error_log.latest_failure("rhoai", "chat") is None
@pytest.mark.asyncio
async def test_a_passing_embedding_call_clears_a_latched_embedding_failure(_healthy_probe):
provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")
response = await provider_health.check_provider_health(test_completion=True, user=None)
assert response.status_code == 200
assert provider_error_log.latest_failure("rhoai", "embedding") is None
@pytest.mark.asyncio
async def test_a_tool_less_completion_leaves_a_latched_chat_failure(monkeypatch, _healthy_probe):
"""The LiteLLM probe (RHOAI and friends) sends a plain completion.
Agent traffic fails on tool calls, which that probe never makes, so it can
pass while every chat turn still fails.
"""
_probes(monkeypatch, chat=PLAIN_COMPLETION, embedding=PLAIN_COMPLETION)
provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
response = await provider_health.check_provider_health(test_completion=True, user=None)
assert response.status_code == 503
assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
@pytest.mark.asyncio
async def test_a_validation_that_called_no_model_clears_nothing(monkeypatch, _healthy_probe):
"""Azure's deployment listing, or a role with no model set, makes no model call."""
_probes(monkeypatch, chat=NO_CALL, embedding=NO_CALL)
provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")
response = await provider_health.check_provider_health(test_completion=True, user=None)
assert response.status_code == 503
assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
assert provider_error_log.latest_failure("rhoai", "embedding") == "embeddings fell over"
@pytest.mark.asyncio
async def test_a_model_free_check_proves_too_little_to_clear_anything(_healthy_probe):
"""Without a completion there is no evidence the provider can serve a call."""
provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
response = await provider_health.check_provider_health(test_completion=False, user=None)
assert response.status_code == 503
assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
@pytest.mark.asyncio
async def test_a_failing_probe_leaves_the_recorded_failure_in_place(monkeypatch, _healthy_probe):
"""A provider that is genuinely broken keeps saying so, in its own words."""
async def boom(**kwargs):
if kwargs.get("llm_model"):
raise RuntimeError("endpoint unreachable")
return PLAIN_COMPLETION
monkeypatch.setattr(provider_health, "validate_provider_setup", boom)
provider_error_log.record_failure("rhoai", "chat", "the real traffic failure")
response = await provider_health.check_provider_health(test_completion=True, user=None)
assert response.status_code == 503
# A real call still beats a probe when both have something to say.
assert provider_error_log.latest_failure("rhoai", "chat") == "the real traffic failure"