Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

25.3 KB
Raw Blame
import base64
import json
import uuid
from collections.abc import Callable
from hashlib import sha256
from itertools import chain, count
from typing import Final

import httpx
import pytest
from integration._support.client import Gateway, eventually, string_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from openai import OpenAI
from pydantic import JsonValue, TypeAdapter

_BACKEND: Final = "gpt-5.4-mini"
_API_KEY: Final = "synthetic-openai-key"
_PROMPT: Final = "Summarize this conversation in one sentence."
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])


def _completion(identity: str, content: str) -> bytes:
    return json.dumps(
        {
            "id": identity,
            "object": "chat.completion",
            "created": 1,
            "model": _BACKEND,
            "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
            "usage": {"prompt_tokens": 19, "completion_tokens": 7, "total_tokens": 26},
        }
    ).encode()


@pytest.mark.covers("providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire")
def test_openai_chat_tool_choice_without_tools_is_not_forwarded(gateway: Gateway) -> None:
    identity: Final = f"openai-toolless-{uuid.uuid4().hex}"

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        assert request.target == "/chat/completions"
        assert request.headers["authorization"] == f"Bearer {_API_KEY}"
        body: Final = _JSON_OBJECT.validate_json(request.body)
        assert body["model"] == _BACKEND
        assert body["messages"] == [{"role": "user", "content": _PROMPT}]
        assert "tool_choice" not in body, body
        assert "tools" not in body, body
        return Reply(body=_completion(identity, "One sentence."))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
        response: Final = gateway.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": _PROMPT}], "tool_choice": "none"},
        )
        assert response.status_code == 200, response.text
        payload: Final = _JSON_OBJECT.validate_json(response.content)
        assert payload["id"] == identity
        assert payload["choices"] == [
            {
                "finish_reason": "stop",
                "index": 0,
                "message": {
                    "role": "assistant",
                    "content": "One sentence.",
                },
                "provider_specific_fields": {},
            }
        ]
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]


def test_azure_gpt_6_bridged_stream_returns_text_and_tool_call_on_one_choice(gateway: Gateway) -> None:
    identity: Final = f"azure-gpt-6-sol-stream-{uuid.uuid4().hex}"
    expected_text: Final = "Let me check the weather."
    events: Final = (
        {
            "type": "response.created",
            "response": {
                "id": "resp_weather",
                "object": "response",
                "created_at": 1,
                "status": "in_progress",
                "model": "gpt-6-sol",
            },
        },
        {
            "type": "response.output_item.added",
            "output_index": 0,
            "item": {
                "id": "msg_weather",
                "type": "message",
                "status": "in_progress",
                "role": "assistant",
                "content": [],
            },
        },
        {
            "type": "response.output_text.delta",
            "item_id": "msg_weather",
            "output_index": 0,
            "content_index": 0,
            "delta": "Let me check ",
        },
        {
            "type": "response.output_text.delta",
            "item_id": "msg_weather",
            "output_index": 0,
            "content_index": 0,
            "delta": "the weather.",
        },
        {
            "type": "response.output_item.done",
            "output_index": 0,
            "item": {
                "id": "msg_weather",
                "type": "message",
                "status": "completed",
                "role": "assistant",
                "content": [{"type": "output_text", "text": expected_text, "annotations": []}],
            },
        },
        {
            "type": "response.output_item.added",
            "output_index": 1,
            "item": {
                "id": "fc_1",
                "type": "function_call",
                "status": "in_progress",
                "call_id": "call_1",
                "name": "get_weather",
                "arguments": "",
            },
        },
        {
            "type": "response.function_call_arguments.delta",
            "item_id": "fc_1",
            "output_index": 1,
            "delta": '{"city":',
        },
        {
            "type": "response.function_call_arguments.delta",
            "item_id": "fc_1",
            "output_index": 1,
            "delta": '"Paris"}',
        },
        {
            "type": "response.output_item.done",
            "output_index": 1,
            "item": {
                "id": "fc_1",
                "type": "function_call",
                "status": "completed",
                "call_id": "call_1",
                "name": "get_weather",
                "arguments": '{"city":"Paris"}',
            },
        },
        {
            "type": "response.completed",
            "response": {
                "id": "resp_weather",
                "object": "response",
                "created_at": 1,
                "status": "completed",
                "model": "gpt-6-sol",
                "output": [
                    {
                        "id": "msg_weather",
                        "type": "message",
                        "status": "completed",
                        "role": "assistant",
                        "content": [{"type": "output_text", "text": expected_text, "annotations": []}],
                    },
                    {
                        "id": "fc_1",
                        "type": "function_call",
                        "status": "completed",
                        "call_id": "call_1",
                        "name": "get_weather",
                        "arguments": '{"city":"Paris"}',
                    },
                ],
                "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
            },
        },
    )
    stream_chunks: Final = tuple(f"data: {json.dumps(event)}\n\n".encode() for event in events)

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        assert request.target == "/openai/responses?api-version=2025-04-01-preview"
        body: Final = _JSON_OBJECT.validate_json(request.body)
        assert body["model"] == "gpt-6-sol"
        return Reply(content_type="text/event-stream", chunks=stream_chunks)

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="azure/gpt-6-sol",
            api_base=wire.url,
            api_key=_API_KEY,
            api_version="2025-04-01-preview",
        )
        with gateway.client.stream(
            "POST",
            "/v1/chat/completions",
            headers={"Authorization": f"Bearer {gateway.key}"},
            json={
                "model": model,
                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
                "tools": [
                    {
                        "type": "function",
                        "function": {
                            "name": "get_weather",
                            "description": "Get the weather for a city.",
                            "parameters": {
                                "type": "object",
                                "properties": {"city": {"type": "string"}},
                                "required": ["city"],
                            },
                        },
                    }
                ],
                "stream": True,
                "cache": {"no-cache": True},
            },
        ) as response:
            response_body: Final = response.read()
            assert response.status_code == 200, response.text
            chunks: Final = tuple(
                _JSON_OBJECT.validate_json(line.removeprefix("data: "))
                for line in response_body.decode().splitlines()
                if line.startswith("data: ") and line != "data: [DONE]"
            )
            choices: Final = tuple(chain.from_iterable(chunk["choices"] for chunk in chunks))
            assert choices, response.text
            assert all(choice["index"] == 0 for choice in choices), response.text
            assert "".join(str(choice["delta"].get("content") or "") for choice in choices) == expected_text, (
                response.text
            )
            tool_call_chunks: Final = tuple(
                chain.from_iterable(choice["delta"].get("tool_calls", []) for choice in choices)
            )
            assert (
                "".join(str(tool_call["function"].get("name") or "") for tool_call in tool_call_chunks) == "get_weather"
            ), response.text
            assert (
                "".join(str(tool_call["function"].get("arguments") or "") for tool_call in tool_call_chunks)
                == '{"city":"Paris"}'
            ), response.text
            assert tuple(
                choice.get("finish_reason") for choice in choices if choice.get("finish_reason") is not None
            ) == ("tool_calls",), response.text
        assert [(request.method, request.target) for request in wire.drain()] == [
            ("POST", "/openai/responses?api-version=2025-04-01-preview")
        ]


_SPEND_ROW: Final = 'SELECT request_tags, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
_TAGGED_METADATA: Final = json.dumps({"tags": ["t1"]})


def _spend_row(request_id: str) -> tuple[list[JsonValue], dict[str, JsonValue]]:
    rows: Final = eventually(lambda: read_rows(_SPEND_ROW, (request_id,)), lambda values: len(values) == 1, seconds=70)
    tags: Final = rows[0]["request_tags"]
    metadata: Final = rows[0]["metadata"]
    assert isinstance(tags, list) and isinstance(metadata, dict), rows
    return tags, metadata


def _openai(gateway: Gateway, key: str) -> OpenAI:
    return OpenAI(
        api_key=key,
        base_url=str(gateway.client.base_url).rstrip("/") + "/v1",
        max_retries=0,
        http_client=httpx.Client(timeout=15, trust_env=False),
    )


def _tagged_chat_wire(identity: str, expected: dict[str, JsonValue]) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        assert (request.method, request.target) == ("POST", "/chat/completions"), request
        assert request.headers["authorization"] == f"Bearer {_API_KEY}"
        assert request.headers["content-type"] == "application/json"
        assert _JSON_OBJECT.validate_json(request.body) == expected
        return Reply(body=_completion(identity, "tagged"))

    return respond


def test_openai_chat_json_string_metadata_reaches_upstream_and_keeps_key_identity(gateway: Gateway) -> None:
    identity: Final = f"openai-string-metadata-{uuid.uuid4().hex}"
    expected: Final = {"model": _BACKEND, "messages": [{"role": "user", "content": identity}]}

    with wire_server(_tagged_chat_wire(identity, expected)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
        team: Final = scenario.team()
        user_id: Final = f"integration-{uuid.uuid4().hex}"
        key: Final = scenario.key(models=[model], team_id=team, user_id=user_id)
        with _openai(gateway, key) as client:
            completion: Final = client.chat.completions.create(
                model=model,
                messages=[{"role": "user", "content": identity}],
                extra_body={"metadata": _TAGGED_METADATA},
            )
        assert (completion.id, completion.model, completion.choices[0].message.content) == (identity, model, "tagged")
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
        _, metadata = _spend_row(identity)
        assert (
            metadata["user_api_key"],
            metadata["user_api_key_user_id"],
            metadata["user_api_key_team_id"],
        ) == (sha256(key.encode()).hexdigest(), user_id, team), metadata


def test_openai_chat_json_string_metadata_tags_the_spend_row(gateway: Gateway) -> None:
    pytest.skip(
        "BUG: /v1/chat/completions replaces JSON-string metadata with {} so the client tag never reaches SpendLogs"
    )
    identity: Final = f"openai-string-metadata-tags-{uuid.uuid4().hex}"
    expected: Final = {"model": _BACKEND, "messages": [{"role": "user", "content": identity}]}

    with wire_server(_tagged_chat_wire(identity, expected)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
        response: Final = gateway.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": identity}], "metadata": _TAGGED_METADATA},
        )
        assert response.status_code == 200, response.text
        assert response.json()["id"] == identity, response.text
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
        tags, _ = _spend_row(identity)
        assert "t1" in tags, tags


def test_openai_chat_multipart_form_fields_reach_upstream_as_json_types(gateway: Gateway) -> None:
    pytest.skip("BUG: multipart /v1/chat/completions returns 400 because messages stays a JSON string")
    identity: Final = f"openai-form-chat-{uuid.uuid4().hex}"
    messages: Final = [{"role": "user", "content": identity}]
    expected: Final = {"model": _BACKEND, "messages": messages, "temperature": 0.2}

    with wire_server(_tagged_chat_wire(identity, expected)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
        fields: Final = {
            "model": model,
            "messages": json.dumps(messages),
            "temperature": "0.2",
            "stream": "false",
            "metadata": _TAGGED_METADATA,
        }
        response: Final = gateway.client.post(
            "/v1/chat/completions",
            files=[(name, (None, value)) for name, value in fields.items()],
            headers={"Authorization": f"Bearer {gateway.key}"},
        )
        assert response.status_code == 200, response.text
        assert response.request.headers["content-type"].startswith("multipart/form-data"), response.request.headers
        payload: Final = _JSON_OBJECT.validate_json(response.content)
        assert (payload["id"], payload["object"]) == (identity, "chat.completion"), response.text
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
        tags, _ = _spend_row(identity)
        assert "t1" in tags, tags


def test_openai_chat_sdk_parameter_set_reaches_upstream_whole(gateway: Gateway) -> None:
    identity: Final = f"openai-full-params-{uuid.uuid4().hex}"
    tools: Final = [
        {
            "type": "function",
            "function": {
                "name": "add",
                "description": "Add two integers",
                "parameters": {
                    "type": "object",
                    "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}},
                    "required": ["a", "b"],
                },
            },
        }
    ]
    response_format: Final = {
        "type": "json_schema",
        "json_schema": {
            "name": "sum",
            "schema": {
                "type": "object",
                "properties": {"total": {"type": "integer"}},
                "required": ["total"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    }
    messages: Final = [{"role": "user", "content": f"Add 3 and 5. {identity}"}]
    parameters: Final = {
        "messages": messages,
        "tools": tools,
        "tool_choice": {"type": "function", "function": {"name": "add"}},
        "parallel_tool_calls": False,
        "response_format": response_format,
        "max_completion_tokens": 64,
        "seed": 7,
        "n": 2,
        "logprobs": True,
        "top_logprobs": 2,
        "user": "integration-end-user",
    }

    def respond(request: Request) -> Reply:
        assert (request.method, request.target) == ("POST", "/chat/completions"), request
        assert request.headers["authorization"] == f"Bearer {_API_KEY}"
        assert _JSON_OBJECT.validate_json(request.body) == {"model": _BACKEND, **parameters}
        return Reply(
            body=json.dumps(
                {
                    "id": identity,
                    "object": "chat.completion",
                    "created": 1,
                    "model": _BACKEND,
                    "choices": [
                        {
                            "index": 0,
                            "message": {"role": "assistant", "content": '{"total": 8}'},
                            "finish_reason": "stop",
                        },
                        {
                            "index": 1,
                            "message": {"role": "assistant", "content": '{"total": 9}'},
                            "finish_reason": "length",
                        },
                    ],
                    "usage": {"prompt_tokens": 19, "completion_tokens": 14, "total_tokens": 33},
                }
            ).encode()
        )

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
        key: Final = scenario.key(models=[model])
        with _openai(gateway, key) as client:
            completion: Final = client.chat.completions.create(model=model, **parameters)
        assert (completion.id, completion.model) == (identity, model), completion
        assert [(choice.index, choice.finish_reason, choice.message.content) for choice in completion.choices] == [
            (0, "stop", '{"total": 8}'),
            (1, "length", '{"total": 9}'),
        ], completion
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]


_RESPONSES_TARGET: Final = "/openai/responses?api-version=2025-04-01-preview"


def _responses_json(identity: str) -> bytes:
    return json.dumps(
        {
            "id": identity,
            "object": "response",
            "created_at": 1,
            "status": "completed",
            "model": "gpt-6-sol",
            "output": [
                {
                    "id": f"msg_{identity}",
                    "type": "message",
                    "status": "completed",
                    "role": "assistant",
                    "content": [{"type": "output_text", "text": "hi", "annotations": []}],
                }
            ],
            "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
        }
    ).encode()


def _gpt_6_function_request(model: str, identity: str, **extra: JsonValue) -> dict[str, JsonValue]:
    return {
        "model": model,
        "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
        "tools": [
            {
                "type": "function",
                "function": {
                    "name": "get_weather",
                    "description": "Get the weather for a city.",
                    "parameters": {
                        "type": "object",
                        "properties": {"city": {"type": "string"}},
                        "required": ["city"],
                    },
                },
            }
        ],
        **extra,
    }


def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_log_spend(
    gateway: Gateway,
) -> None:
    identity: Final = f"azure-gpt-6-sol-nocache-{uuid.uuid4().hex}"
    response_ids: Final = ("resp_first", "resp_second")
    calls: Final = count()

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        assert request.target == _RESPONSES_TARGET
        body: Final = _JSON_OBJECT.validate_json(request.body)
        assert body["model"] == "gpt-6-sol"
        assert body["input"] == [
            {
                "type": "message",
                "role": "user",
                "content": [{"type": "input_text", "text": f"What is the weather in Paris? {identity}"}],
            }
        ]
        return Reply(body=_responses_json(response_ids[next(calls)]))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="azure/gpt-6-sol",
            api_base=wire.url,
            api_key=_API_KEY,
            api_version="2025-04-01-preview",
            input_cost_per_token=0.001,
            output_cost_per_token=0.002,
        )
        request: Final = _gpt_6_function_request(model, identity, cache={"no-cache": True})
        first: Final = gateway.request("POST", "/v1/chat/completions", request)
        second: Final = gateway.request("POST", "/v1/chat/completions", request)
        assert first.status_code == 200, first.text
        assert second.status_code == 200, second.text
        assert string_value(_JSON_OBJECT.validate_json(first.content)["id"]) == "resp_first", first.text
        assert string_value(_JSON_OBJECT.validate_json(second.content)["id"]) == "resp_second", second.text
        assert [(request.method, request.target) for request in wire.drain()] == [
            ("POST", _RESPONSES_TARGET),
            ("POST", _RESPONSES_TARGET),
        ]
        rows: Final = eventually(
            lambda: read_rows(
                'SELECT request_id, status, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
                (model,),
            ),
            lambda found: len(found) == 2,
            seconds=70,
        )
        by_response_id: Final = {
            (decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode())
            .rsplit("response_id:", 1)[1]: (decoded, row)
            for row in rows
        }
        for response_id in response_ids:
            decoded, row = by_response_id[response_id]
            assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
            assert (
                string_value(row["status"]),
                string_value(row["cache_hit"]),
                float(row["spend"]),
            ) == ("success", "None", pytest.approx(10 * 0.001 + 5 * 0.002)), rows


def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cache(
    gateway: Gateway,
) -> None:
    identity: Final = f"azure-gpt-6-sol-cached-{uuid.uuid4().hex}"

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        assert request.target == _RESPONSES_TARGET
        body: Final = _JSON_OBJECT.validate_json(request.body)
        assert body["model"] == "gpt-6-sol"
        return Reply(body=_responses_json("resp_cached"))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="azure/gpt-6-sol",
            api_base=wire.url,
            api_key=_API_KEY,
            api_version="2025-04-01-preview",
            input_cost_per_token=0.001,
            output_cost_per_token=0.002,
        )
        request: Final = _gpt_6_function_request(model, identity)
        first: Final = gateway.request("POST", "/v1/chat/completions", request)
        second: Final = gateway.request("POST", "/v1/chat/completions", request)
        assert first.status_code == 200, first.text
        assert second.status_code == 200, second.text
        assert string_value(_JSON_OBJECT.validate_json(first.content)["id"]) == "resp_cached", first.text
        assert string_value(_JSON_OBJECT.validate_json(second.content)["id"]) == "resp_cached", second.text
        assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_TARGET)]
        rows: Final = eventually(
            lambda: read_rows(
                'SELECT request_id, status, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE model_group=%s'
                " ORDER BY request_id",
                (model,),
            ),
            lambda found: len(found) == 2,
            seconds=70,
        )
        priced, cached = rows
        priced_request: Final = base64.b64decode(
            string_value(priced["request_id"]).removeprefix("resp_")
        ).decode()
        assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
        assert priced_request.endswith(";response_id:resp_cached"), rows
        assert (
            priced["status"],
            priced["cache_hit"],
            float(priced["spend"]),
        ) == ("success", "None", pytest.approx(10 * 0.001 + 5 * 0.002)), rows
        assert string_value(cached["request_id"]).startswith("resp_cached_cache_hit"), rows
        assert (cached["status"], cached["cache_hit"], float(cached["spend"])) == ("success", "True", 0), rows