The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from collections.abc import Iterator
from contextlib import contextmanager
from typing import Final
import httpx
import openai
from tests.integration._support.client import Gateway
from tests.integration._support.wire import Reply, Request, wire_server
_CHAT_CHUNKS: Final = (
{
"id": "chatcmpl-logprobs",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": "hello"},
"logprobs": {
"content": [
{
"token": "hello",
"logprob": -0.1,
"bytes": [104, 101, 108, 108, 111],
"top_logprobs": [
{"token": "hello", "logprob": -0.1, "bytes": [104, 101, 108, 108, 111]},
{"token": "hi", "logprob": -2.3, "bytes": [104, 105]},
],
}
]
},
"finish_reason": None,
}
],
},
{
"id": "chatcmpl-logprobs",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": "stop"}],
},
)
_COMPLETION_CHUNKS: Final = (
{
"id": "cmpl-usage",
"object": "text_completion",
"created": 1,
"model": "gpt-3.5-turbo-instruct",
"choices": [{"text": "done", "index": 0, "logprobs": None, "finish_reason": None}],
},
{
"id": "cmpl-usage",
"object": "text_completion",
"created": 1,
"model": "gpt-3.5-turbo-instruct",
"choices": [{"text": "", "index": 0, "logprobs": None, "finish_reason": "stop"}],
},
{
"id": "cmpl-usage",
"object": "text_completion",
"created": 1,
"model": "gpt-3.5-turbo-instruct",
"choices": [],
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
},
)
def _event_stream(chunks: tuple[dict[str, object], ...]) -> Reply:
return Reply(
chunks=(*(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks), b"data: [DONE]\n\n"),
content_type="text/event-stream",
)
@contextmanager
def _client(gateway: Gateway) -> Iterator[openai.OpenAI]:
with (
httpx.Client(timeout=15, trust_env=False) as transport,
openai.OpenAI(
api_key=gateway.key, base_url=str(gateway.client.base_url.join("/v1")), max_retries=0, http_client=transport
) as client,
):
yield client
def test_chat_stream_relays_the_provider_logprobs(gateway: Gateway) -> None:
def upstream(request: Request) -> Reply:
return _event_stream(_CHAT_CHUNKS)
with wire_server(upstream) as provider, gateway.scenario() as scenario, _client(gateway) as client:
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=f"{provider.url}/v1")
chunks: Final = tuple(
client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": "Hello!"}],
logprobs=True,
top_logprobs=2,
stream=True,
)
)
sent: Final = provider.drain()
assert len(sent) == 1
outbound: Final = json.loads(sent[0].body)
assert (outbound["logprobs"], outbound["top_logprobs"], outbound["stream"]) == (True, 2, True)
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == "hello"
relayed: Final = tuple(
chunk.choices[0].logprobs.model_dump(exclude_none=True)
for chunk in chunks
if chunk.choices and chunk.choices[0].logprobs is not None
)
assert relayed == (_CHAT_CHUNKS[0]["choices"][0]["logprobs"],)
def test_completion_stream_reports_the_provider_usage_when_include_usage_is_set(gateway: Gateway) -> None:
def upstream(request: Request) -> Reply:
return _event_stream(_COMPLETION_CHUNKS)
with wire_server(upstream) as provider, gateway.scenario() as scenario, _client(gateway) as client:
model: Final = scenario.model(model="openai/gpt-3.5-turbo-instruct", api_base=f"{provider.url}/v1")
chunks: Final = tuple(
client.completions.create(
model=model,
prompt="hey",
stream=True,
stream_options={"include_usage": True},
max_tokens=4,
)
)
sent: Final = provider.drain()
assert len(sent) == 1
outbound: Final = json.loads(sent[0].body)
assert (outbound["prompt"], outbound["stream_options"], outbound["max_tokens"]) == (
"hey",
{"include_usage": True},
4,
)
assert "".join(choice.text or "" for chunk in chunks for choice in chunk.choices) == "done"
usages: Final = tuple(
(chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
for chunk in chunks
if chunk.usage is not None
)
assert usages == ((7, 3, 10),)