OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""OpenAI-compatible /v1 LLM proxy handlers."""
import inspect
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import params as fastapi_params
from fastapi.responses import JSONResponse, StreamingResponse
from api.v1 import llm as v1_llm
from services.llm_gateway import LlmContextLengthError, LlmGatewayError
from services.model_catalog import CATALOG_UNAVAILABLE_MESSAGE, CatalogUnavailableError
from session_manager import User
def _user() -> User:
return User(user_id="u1", email="u@x", name="U", provider="api_key")
def _get_user_dependency(fn):
user_param = inspect.signature(fn).parameters["user"]
default = user_param.default
assert isinstance(default, fastapi_params.Depends)
return default.dependency
def test_completions_requires_chat_use():
dep = _get_user_dependency(v1_llm.chat_completions_endpoint)
perms = [cell.cell_contents for cell in dep.__closure__]
assert "chat:use" in perms
def test_embeddings_accepts_chat_or_upload():
dep = _get_user_dependency(v1_llm.embeddings_endpoint)
required = [cell.cell_contents for cell in dep.__closure__]
assert any("chat:use" in str(item) or "knowledge:upload" in str(item) for item in required)
@pytest.mark.asyncio
async def test_list_openai_models_returns_openai_list():
response = await v1_llm.list_openai_models_endpoint(user=_user())
assert isinstance(response, JSONResponse)
data = json.loads(response.body)
assert data["object"] == "list"
assert data["data"]
assert data["data"][0]["object"] == "model"
@pytest.mark.asyncio
async def test_model_catalog_returns_providers():
response = await v1_llm.model_catalog_endpoint(user=_user())
data = json.loads(response.body)
keys = {p["key"] for p in data["providers"]}
assert "openai" in keys
assert "anthropic" in keys
openai = next(p for p in data["providers"] if p["key"] == "openai")
assert openai["models"]
assert openai["embedding_models"]
assert openai["credential_fields"]
@pytest.mark.asyncio
async def test_chat_completions_delegates_to_gateway(monkeypatch):
monkeypatch.setattr(
v1_llm,
"chat_completions",
AsyncMock(
return_value={
"id": "chatcmpl-1",
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
}
),
)
request = MagicMock()
request.json = AsyncMock(
return_value={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}
)
response = await v1_llm.chat_completions_endpoint(request, user=_user())
data = json.loads(response.body)
assert data["choices"][0]["message"]["content"] == "ok"
@pytest.mark.asyncio
async def test_chat_completions_streams(monkeypatch):
async def fake_stream():
yield 'data: {"choices":[]}\n\n'
yield "data: [DONE]\n\n"
monkeypatch.setattr(v1_llm, "chat_completions", AsyncMock(return_value=fake_stream()))
request = MagicMock()
request.json = AsyncMock(return_value={"model": "gpt-4o-mini", "messages": [], "stream": True})
response = await v1_llm.chat_completions_endpoint(request, user=_user())
assert isinstance(response, StreamingResponse)
assert response.media_type == "text/event-stream"
@pytest.mark.asyncio
async def test_chat_completions_maps_gateway_errors(monkeypatch):
"""Return gateway failures without inventing a code for an unclassified error."""
monkeypatch.setattr(
v1_llm,
"chat_completions",
AsyncMock(side_effect=LlmGatewayError("OpenAI API key is not configured", 400)),
)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "gpt-4o-mini", "messages": []})
response = await v1_llm.chat_completions_endpoint(request, user=_user())
assert response.status_code == 400
data = json.loads(response.body)
assert data["error"]["message"] == "OpenAI API key is not configured"
# No code is invented for a failure the gateway did not classify.
assert "code" not in data["error"]
@pytest.mark.asyncio
async def test_embeddings_reports_an_over_length_input_as_a_coded_400(monkeypatch):
"""What an OpenAI client expects for it, so it can branch on `error.code`."""
message = "watsonx/e5: This model's maximum context length is 512 tokens."
monkeypatch.setattr(
v1_llm,
"embeddings",
AsyncMock(side_effect=LlmContextLengthError(message, detail="BadRequestError: ...")),
)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "watsonx:e5", "input": ["x"]})
response = await v1_llm.embeddings_endpoint(request, user=_user())
assert response.status_code == 400
assert json.loads(response.body) == {
"error": {
"message": message,
"type": "invalid_request_error",
"code": "context_length_exceeded",
}
}
@pytest.mark.asyncio
async def test_a_classified_upstream_failure_carries_its_code(monkeypatch):
"""Preserve the classified upstream status and code in an embeddings response."""
monkeypatch.setattr(
v1_llm,
"embeddings",
AsyncMock(
side_effect=LlmGatewayError("rate limited", 429, code="rate_limit_exceeded"),
),
)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "watsonx:e5", "input": ["x"]})
response = await v1_llm.embeddings_endpoint(request, user=_user())
assert response.status_code == 429
assert json.loads(response.body)["error"]["code"] == "rate_limit_exceeded"
@pytest.mark.asyncio
async def test_embeddings_delegates_to_gateway(monkeypatch):
monkeypatch.setattr(
v1_llm,
"embeddings",
AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]}),
)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
response = await v1_llm.embeddings_endpoint(request, user=_user())
data = json.loads(response.body)
assert data["data"][0]["embedding"] == [0.2]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("purpose", "interactive"),
[("chat", True), ("ingest", False), (None, False)],
)
async def test_embeddings_lane_follows_the_hop_token_purpose(monkeypatch, purpose, interactive):
gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})
monkeypatch.setattr(v1_llm, "embeddings", gateway)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})
request.state = SimpleNamespace(llm_hop_purpose=purpose)
await v1_llm.embeddings_endpoint(request, user=_user())
assert gateway.await_args.kwargs == {"interactive": interactive}
@pytest.mark.asyncio
async def test_embeddings_without_a_hop_token_take_the_bulk_lane(monkeypatch):
gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})
monkeypatch.setattr(v1_llm, "embeddings", gateway)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})
request.state = SimpleNamespace() # API key / JWT: no hop purpose recorded
await v1_llm.embeddings_endpoint(request, user=_user())
assert gateway.await_args.kwargs == {"interactive": False}
@pytest.mark.asyncio
async def test_list_openai_models_hides_catalog_exception_text(monkeypatch):
"""CodeQL py/stack-trace-exposure: the exception text must not reach the caller."""
monkeypatch.setattr(
v1_llm,
"openai_models_list",
MagicMock(side_effect=CatalogUnavailableError("litellm is not installed on the server")),
)
response = await v1_llm.list_openai_models_endpoint(user=_user())
assert response.status_code == 503
data = json.loads(response.body)
assert data["error"]["message"] == CATALOG_UNAVAILABLE_MESSAGE
assert "litellm" not in response.body.decode()
@pytest.mark.asyncio
async def test_model_catalog_hides_catalog_exception_text(monkeypatch):
monkeypatch.setattr(
v1_llm,
"catalog",
MagicMock(side_effect=CatalogUnavailableError("litellm is not installed on the server")),
)
response = await v1_llm.model_catalog_endpoint(user=_user())
assert response.status_code == 503
data = json.loads(response.body)
assert data["error"] == CATALOG_UNAVAILABLE_MESSAGE
assert "litellm" not in response.body.decode()
@pytest.mark.asyncio
async def test_gateway_error_detail_is_not_returned_to_the_caller(monkeypatch):
"""`detail` is log-only; only the sanitized `message` is serialized."""
monkeypatch.setattr(
v1_llm,
"chat_completions",
AsyncMock(
side_effect=LlmGatewayError(
"The model provider could not be reached. Please try again.",
502,
detail="RuntimeError: connect to 10.0.0.5 failed",
)
),
)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "gpt-4o-mini", "messages": []})
response = await v1_llm.chat_completions_endpoint(request, user=_user())
assert response.status_code == 502
body = response.body.decode()
assert "10.0.0.5" not in body
assert "RuntimeError" not in body
assert json.loads(body)["error"]["type"] == "api_error"
# ---------------------------------------------------------------------------
# Tracing: who a /v1 call is for
# ---------------------------------------------------------------------------
@pytest.fixture
def tracing_on(monkeypatch):
from observability import tracing
monkeypatch.setattr(tracing, "_enabled", True)
def _metadata_seen_by_gateway(kind: str):
"""A fake gateway call that records the trace metadata it would send."""
from observability.tracing import gateway_trace_kwargs
seen = {}
async def fake(*args, **kwargs):
seen.update(gateway_trace_kwargs(kind, "ingest")["metadata"])
if kind == "chat":
return {"choices": [{"message": {"role": "assistant", "content": "ok"}}]}
return {"object": "list", "data": [{"embedding": [0.2]}]}
return fake, seen
@pytest.mark.asyncio
async def test_langflow_embedding_calls_are_traced_to_the_user_and_run(monkeypatch, tracing_on):
fake, seen = _metadata_seen_by_gateway("embedding")
monkeypatch.setattr(v1_llm, "embeddings", fake)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
request.state = SimpleNamespace(llm_hop_purpose="ingest", llm_hop_run_id="run-1")
await v1_llm.embeddings_endpoint(request, user=_user())
assert seen["trace_metadata"]["openrag.source"] == "langflow"
assert seen["trace_metadata"]["openrag.langflow_run_purpose"] == "ingest"
assert seen["trace_user_id"] == "u1"
assert seen["session_id"] == "run-1"
@pytest.mark.asyncio
async def test_langflow_chat_calls_are_traced_to_the_user_and_run(monkeypatch, tracing_on):
fake, seen = _metadata_seen_by_gateway("chat")
monkeypatch.setattr(v1_llm, "chat_completions", fake)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "gpt-4o-mini", "messages": []})
request.state = SimpleNamespace(llm_hop_purpose="chat", llm_hop_run_id="run-2")
await v1_llm.chat_completions_endpoint(request, user=_user())
assert seen["trace_metadata"]["openrag.source"] == "langflow"
assert seen["trace_user_id"] == "u1"
assert seen["session_id"] == "run-2"
@pytest.mark.asyncio
async def test_api_key_calls_are_traced_as_api_without_a_session(monkeypatch, tracing_on):
fake, seen = _metadata_seen_by_gateway("embedding")
monkeypatch.setattr(v1_llm, "embeddings", fake)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
request.state = SimpleNamespace() # API key / JWT: not a Langflow run
await v1_llm.embeddings_endpoint(request, user=_user())
assert seen["trace_metadata"]["openrag.source"] == "api"
assert seen["trace_user_id"] == "u1"
assert "session_id" not in seen
assert "openrag.langflow_run_purpose" not in seen["trace_metadata"]
@pytest.mark.asyncio
async def test_api_key_embedding_calls_get_their_own_purpose(monkeypatch, tracing_on):
"""Not every non-interactive embedding is ingestion: an API caller's is labelled as such."""
fake, seen = _metadata_seen_by_gateway("embedding")
monkeypatch.setattr(v1_llm, "embeddings", fake)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
request.state = SimpleNamespace()
await v1_llm.embeddings_endpoint(request, user=_user())
assert seen["generation_name"] == "embedding.api"
assert seen["trace_metadata"]["openrag.purpose"] == "api"
@pytest.mark.asyncio
async def test_langflow_ingest_embedding_calls_stay_ingest(monkeypatch, tracing_on):
fake, seen = _metadata_seen_by_gateway("embedding")
monkeypatch.setattr(v1_llm, "embeddings", fake)
request = MagicMock()
request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
request.state = SimpleNamespace(llm_hop_purpose="ingest", llm_hop_run_id="run-1")
await v1_llm.embeddings_endpoint(request, user=_user())
assert seen["generation_name"] == "embedding.ingest"