Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

20.6 KB
Raw Blame
from __future__ import annotations

import json
import uuid
from typing import Final, Literal

import anthropic
import httpx
import openai
import pytest
import websockets
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue


def _marker() -> str:
    return uuid.uuid4().hex


def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
    with httpx.Client(timeout=5, trust_env=False) as client:
        result: Final = client.post(
            f"{gateway.upstream_url}/__scenarios",
            json={"scenario_id": scenario_id, "response": response},
        )
    assert result.status_code == 200, result.text


_ANTHROPIC_MESSAGE: Final = {
    "id": "msg_$UNIQUE_ID",
    "type": "message",
    "role": "assistant",
    "model": "claude-3-5-sonnet-20241022",
    "content": [{"type": "text", "text": "scripted reply"}],
    "stop_reason": "end_turn",
    "usage": {"input_tokens": 9, "output_tokens": 5},
}

_ANTHROPIC_EVENTS: Final = (
    'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}',
    'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
    'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted reply"}}',
    'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
    'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}',
    'event: message_stop\ndata: {"type":"message_stop"}',
)

_RESPONSES_BODY: Final = {
    "id": "resp_$UNIQUE_ID",
    "object": "response",
    "created_at": 1700000000,
    "status": "completed",
    "model": "openai/gpt-4o-mini",
    "output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
    "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}

_RESPONSES_EVENTS: Final = (
    'event: response.created\ndata: {"type":"response.created","response":{"id":"$UNIQUE_ID","object":"response","status":"in_progress"}}',
    'event: response.completed\ndata: {"type":"response.completed","response":{"id":"$UNIQUE_ID","object":"response","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
)


def _anthropic_model(scenario: Scenario, *, stream: bool) -> str:
    gateway: Final = scenario.gateway
    scenario_id: Final = "audit-anthropic-stream" if stream else "audit-anthropic"
    if stream:
        _register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS})
    else:
        _register_scenario(
            gateway,
            scenario_id,
            {
                "content_type": "application/x-routed",
                "routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}},
            },
        )
    return scenario.model(
        model="anthropic/claude-3-5-sonnet-20241022", api_base=f"{gateway.upstream_url}/{scenario_id}"
    )


def _responses_model(scenario: Scenario, *, stream: bool) -> str:
    gateway: Final = scenario.gateway
    scenario_id: Final = "audit-resp-stream" if stream else "audit-resp"
    if stream:
        _register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS})
    else:
        _register_scenario(
            gateway,
            scenario_id,
            {
                "content_type": "application/x-routed",
                "routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}},
            },
        )
    return scenario.model(api_base=f"{gateway.upstream_url}/{scenario_id}")


def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI:
    return openai.OpenAI(
        base_url=f"{gateway.client.base_url}/v1",
        api_key=key,
        http_client=httpx.Client(trust_env=False),
    )


def _openai_async_client(gateway: Gateway, key: str) -> openai.AsyncOpenAI:
    return openai.AsyncOpenAI(
        base_url=f"{gateway.client.base_url}/v1",
        api_key=key,
        http_client=httpx.AsyncClient(trust_env=False),
    )


def _anthropic_client(gateway: Gateway, key: str) -> anthropic.Anthropic:
    return anthropic.Anthropic(
        base_url=str(gateway.client.base_url).rstrip("/"),
        api_key=key,
        http_client=httpx.Client(trust_env=False),
    )


def _anthropic_async_client(gateway: Gateway, key: str) -> anthropic.AsyncAnthropic:
    return anthropic.AsyncAnthropic(
        base_url=str(gateway.client.base_url).rstrip("/"),
        api_key=key,
        http_client=httpx.AsyncClient(trust_env=False),
    )


def _sse_chunk_models(text: str) -> tuple[str, ...]:
    models: list[str] = []
    for line in text.splitlines():
        if not line.startswith("data: ") or line == "data: [DONE]":
            continue
        value = json.loads(line.removeprefix("data: "))
        if isinstance(value, dict) and isinstance(value.get("model"), str):
            models.append(value["model"])
    return tuple(models)


@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_chat_completions_restamps_alias(
    gateway: Gateway, client_kind: Literal["httpx", "openai", "openai_async"], stream: bool
) -> None:
    with gateway.scenario() as scenario:
        model: Final = scenario.model()
        key: Final = scenario.key(models=[model])
        body: Final = {
            "model": model,
            "messages": [{"role": "user", "content": _marker()}],
            "stream": stream,
            **({"stream_options": {"include_usage": True}} if stream else {}),
        }
        if client_kind == "httpx":
            response: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
            assert response.status_code == 200, response.text
            if stream:
                models: Final = _sse_chunk_models(response.text)
                assert models and all(chunk_model == model for chunk_model in models), response.text
            else:
                assert response.json()["model"] == model, response.text
        elif client_kind == "openai":
            client: Final = _openai_client(gateway, key)
            completion: Final = client.chat.completions.create(**body)
            if stream:
                chunk_models: list[str] = []
                for chunk in completion:
                    if chunk.model:
                        chunk_models.append(chunk.model)
                assert chunk_models and all(chunk_model == model for chunk_model in chunk_models)
            else:
                assert completion.model == model
        else:

            async def _run() -> None:
                client_async: Final = _openai_async_client(gateway, key)
                completion_async: Final = await client_async.chat.completions.create(**body)
                if stream:
                    seen: list[str] = []
                    async for chunk in completion_async:
                        if chunk.model:
                            seen.append(chunk.model)
                    assert seen and all(chunk_model == model for chunk_model in seen)
                else:
                    assert completion_async.model == model

            import asyncio

            asyncio.run(_run())


@pytest.mark.parametrize("client_kind", ("httpx", "anthropic", "anthropic_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_messages(gateway: Gateway, client_kind: str, stream: bool) -> None:
    with gateway.scenario() as scenario:
        model: Final = _anthropic_model(scenario, stream=stream)
        key: Final = scenario.key(models=[model])
        body: Final = {
            "model": model,
            "max_tokens": 64,
            "messages": [{"role": "user", "content": _marker()}],
            "stream": stream,
        }
        if client_kind == "httpx":
            response: Final = gateway.request("POST", "/v1/messages", body, key=key)
            assert response.status_code == 200, response.text
            if stream:
                assert "message_start" in response.text and "message_stop" in response.text, response.text
            else:
                assert response.json()["type"] == "message", response.text
        elif client_kind == "anthropic":
            client: Final = _anthropic_client(gateway, key)
            if stream:
                with client.messages.stream(**{k: v for k, v in body.items() if k != "stream"}) as event_stream:
                    message: Final = event_stream.get_final_message()
                assert message.type == "message" and message.content, message
            else:
                message = client.messages.create(**{k: v for k, v in body.items() if k != "stream"})
                assert message.type == "message" and message.content, message
        else:

            async def _run() -> None:
                client_async: Final = _anthropic_async_client(gateway, key)
                params: Final = {k: v for k, v in body.items() if k != "stream"}
                if stream:
                    async with client_async.messages.stream(**params) as event_stream:
                        message_async: Final = await event_stream.get_final_message()
                else:
                    message_async = await client_async.messages.create(**params)
                assert message_async.type == "message" and message_async.content, message_async

            import asyncio

            asyncio.run(_run())


@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_responses(gateway: Gateway, client_kind: str, stream: bool) -> None:
    with gateway.scenario() as scenario:
        model: Final = _responses_model(scenario, stream=stream)
        key: Final = scenario.key(models=[model])
        body: Final = {
            "model": model,
            "input": [{"role": "user", "content": [{"type": "input_text", "text": _marker()}]}],
            "stream": stream,
        }
        if client_kind == "httpx":
            response: Final = gateway.request("POST", "/v1/responses", body, key=key)
            assert response.status_code == 200, response.text
            if stream:
                assert "response.completed" in response.text, response.text
            else:
                assert response.json()["object"] == "response", response.text
        elif client_kind == "openai":
            client: Final = _openai_client(gateway, key)
            result: Final = client.responses.create(**body)
            if stream:
                events: list[str] = []
                for event in result:
                    events.append(event.type)
                assert "response.completed" in events, events
            else:
                assert result.object == "response", result
        else:

            async def _run() -> None:
                client_async: Final = _openai_async_client(gateway, key)
                result_async: Final = await client_async.responses.create(**body)
                if stream:
                    events_async: list[str] = []
                    async for event in result_async:
                        events_async.append(event.type)
                    assert "response.completed" in events_async, events_async
                else:
                    assert result_async.object == "response", result_async

            import asyncio

            asyncio.run(_run())


def test_denied_key_rejected(gateway: Gateway) -> None:
    with gateway.scenario() as scenario:
        model: Final = scenario.model()
        other_model: Final = scenario.model()
        key: Final = scenario.key(models=[other_model])
        for path, body in (
            ("/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "x"}]}),
            ("/v1/messages", {"model": model, "max_tokens": 4, "messages": [{"role": "user", "content": "x"}]}),
            ("/v1/responses", {"model": model, "input": "x"}),
        ):
            response: Final = gateway.request("POST", path, body, key=key)
            assert response.status_code in (401, 403), f"{path}: {response.status_code} {response.text}"


async def test_realtime_websocket_allowed_and_denied(gateway: Gateway) -> None:
    rt_scenario: Final = f"auditrt{uuid.uuid4().hex[:8]}"
    _register_scenario(
        gateway,
        rt_scenario,
        {
            "content_type": "application/x-realtime",
            "session_model": "gpt-4o-realtime-preview",
            "events": (
                {"type": "response.done", "response": {"id": "resp_$UNIQUE_ID", "model": "gpt-4o-realtime-preview"}},
            ),
        },
    )
    with gateway.scenario() as scenario:
        name: Final = scenario.model(
            model="openai/gpt-4o-realtime-preview", api_key=rt_scenario, api_base=gateway.upstream_url
        )
        key: Final = scenario.key(models=[name])
        other_model: Final = scenario.model()
        denied_key: Final = scenario.key(models=[other_model])
        url: Final = f"{str(gateway.client.base_url).replace('http://', 'ws://')}/v1/realtime?model={name}"
        async with websockets.connect(url, additional_headers={"Authorization": f"Bearer {key}"}) as websocket:
            session: Final = json.loads(await websocket.recv())
            assert session["type"] == "session.created", session
            assert session["session"]["model"] == "gpt-4o-realtime-preview", session
            await websocket.send(json.dumps({"type": "response.create"}))
            event: Final = json.loads(await websocket.recv())
            assert event["type"] == "response.done", event
        with pytest.raises(websockets.exceptions.InvalidHandshake):
            async with websockets.connect(
                url, additional_headers={"Authorization": f"Bearer {denied_key}"}
            ) as denied_socket:
                await denied_socket.recv()


def test_max_budget_allows_then_denies_and_records_spend(gateway: Gateway) -> None:
    with gateway.scenario() as scenario:
        model: Final = scenario.model()
        key: Final = scenario.key(models=[model], max_budget=0.0000001)
        body: Final = {"model": model, "messages": [{"role": "user", "content": _marker()}]}
        first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
        assert first.status_code == 200, first.text
        request_id: Final = first.json()["id"]
        rows: Final = eventually(
            lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (request_id,)),
            lambda values: len(values) == 1,
            seconds=70,
        )
        assert rows, first.text
        second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
        assert second.status_code == 422, second.text
        assert "budget_exceeded" in second.text, second.text


_PATH_MODEL_BACKEND: Final = "gpt-5.4-mini"
_PATH_MODEL_ROUTES: Final = (
    ("/openai/deployments/{model}/chat/completions", "chat"),
    ("/engines/{model}/chat/completions", "chat"),
    ("/openai/deployments/{model}/completions", "text"),
    ("/engines/{model}/completions", "text"),
)


def _path_model_reply(request: Request) -> Reply:
    assert (request.method, request.target) == ("POST", "/chat/completions"), request
    return Reply(
        body=json.dumps(
            {
                "id": "chatcmpl-" + _marker(),
                "object": "chat.completion",
                "created": 1,
                "model": _PATH_MODEL_BACKEND,
                "choices": [{"index": 0, "message": {"role": "assistant", "content": "path"}, "finish_reason": "stop"}],
                "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
            }
        ).encode()
    )


def _path_model_body(kind: str, prompt: str) -> dict[str, JsonValue]:
    return {"messages": [{"role": "user", "content": prompt}]} if kind == "chat" else {"prompt": prompt}


def _upstream_bodies(wire: Wire) -> list[dict[str, JsonValue]]:
    return [json.loads(request.body) for request in wire.drain()]


@pytest.mark.parametrize(("route", "kind"), _PATH_MODEL_ROUTES)
def test_path_only_model_alias_refuses_an_unallowed_deployment(gateway: Gateway, route: str, kind: str) -> None:
    with wire_server(_path_model_reply) as allowed_wire, wire_server(_path_model_reply) as other_wire:
        with gateway.scenario() as scenario:
            allowed: Final = scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=allowed_wire.url)
            other: Final = scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=other_wire.url)
            key: Final = scenario.key(models=[allowed])
            response: Final = gateway.request(
                "POST",
                route.format(model=other),
                _path_model_body(kind, _marker()),
                key=key,
                params={"api-version": "2024-10-21"},
            )
            assert (_upstream_bodies(other_wire), _upstream_bodies(allowed_wire)) == ([], []), response.text
            assert response.status_code == 403, response.text
            assert f"The requested model '{other}' is not available for this API key" in response.text, response.text


@pytest.mark.parametrize(("route", "kind"), _PATH_MODEL_ROUTES)
def test_mixed_body_and_path_model_is_refused_or_served_by_the_body_model(
    gateway: Gateway, route: str, kind: str
) -> None:
    pytest.skip(
        "BUG: a key scoped to model A that sends model A in the body and an unallowed model B in the "
        "/openai/deployments or /engines path is served by deployment B"
    )
    with wire_server(_path_model_reply) as allowed_wire, wire_server(_path_model_reply) as other_wire:
        with gateway.scenario() as scenario:
            allowed: Final = scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=allowed_wire.url)
            other: Final = scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=other_wire.url)
            key: Final = scenario.key(models=[allowed])
            prompt: Final = _marker()
            response: Final = gateway.request(
                "POST",
                route.format(model=other),
                {"model": allowed, **_path_model_body(kind, prompt)},
                key=key,
                params={"api-version": "2024-10-21"},
            )
            other_bodies: Final = _upstream_bodies(other_wire)
            allowed_bodies: Final = _upstream_bodies(allowed_wire)
            outcome: Final = (response.status_code, other_bodies, allowed_bodies)
            refused: Final = outcome[0] in (401, 403) and outcome[1:] == ([], [])
            served_by_body_model: Final = outcome == (
                200,
                [],
                [{"model": _PATH_MODEL_BACKEND, "messages": [{"role": "user", "content": prompt}]}],
            )
            assert refused or served_by_body_model, (outcome, response.text)


@pytest.mark.parametrize(("route", "kind"), _PATH_MODEL_ROUTES)
def test_path_only_model_alias_reaches_the_allowed_deployment(gateway: Gateway, route: str, kind: str) -> None:
    with wire_server(_path_model_reply) as allowed_wire, wire_server(_path_model_reply) as other_wire:
        with gateway.scenario() as scenario:
            allowed: Final = scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=allowed_wire.url)
            scenario.model(model=f"openai/{_PATH_MODEL_BACKEND}", api_base=other_wire.url)
            key: Final = scenario.key(models=[allowed])
            prompt: Final = _marker()
            response: Final = gateway.request(
                "POST",
                route.format(model=allowed),
                _path_model_body(kind, prompt),
                key=key,
                params={"api-version": "2024-10-21"},
            )
            assert response.status_code == 200, response.text
            payload: Final = response.json()
            assert [
                choice["message"]["content"] if kind == "chat" else choice["text"] for choice in payload["choices"]
            ] == ["path"], response.text
            assert payload["object"] == ("chat.completion" if kind == "chat" else "text_completion"), response.text
            assert _upstream_bodies(other_wire) == [], response.text
            assert _upstream_bodies(allowed_wire) == [
                {"model": _PATH_MODEL_BACKEND, "messages": [{"role": "user", "content": prompt}]}
            ], response.text