The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "grok-test"
_API_KEY: Final = "synthetic-xai-key"
_REPLY_TEXT: Final = "server tool usage control"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PLAIN_USAGE: Final[dict[str, JsonValue]] = {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}
_ABSENT: Final = "absent"
def _reply(usage: JsonValue) -> bytes:
return json.dumps(
{
"id": "chatcmpl-xai-usage",
"object": "chat.completion",
"created": 1,
"model": _BACKEND,
"choices": [
{"index": 0, "message": {"role": "assistant", "content": _REPLY_TEXT}, "finish_reason": "stop"}
],
"usage": usage,
}
).encode()
def _peer(usage: JsonValue):
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/chat/completions", request.target
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
assert _JSON_OBJECT.validate_json(request.body)["messages"] == [
{"role": "user", "content": "synthetic usage request"}
], request.body
return Reply(body=_reply(usage))
return respond
def _usage_with(details: JsonValue) -> dict[str, JsonValue]:
return {**_PLAIN_USAGE, "server_side_tool_usage_details": details}
@pytest.mark.parametrize(
("usage", "expected_details", "expected_web_search_requests"),
(
pytest.param(
_usage_with({"web_search_calls": 2, "x_search_calls": 1}),
{"web_search_calls": 2, "x_search_calls": 1},
2,
id="web-search-calls-counted",
),
pytest.param(_usage_with({"x_search_calls": 3}), {"x_search_calls": 3}, None, id="no-web-search-calls"),
pytest.param(
_usage_with({"web_search_calls": "many"}), {"web_search_calls": "many"}, None, id="call-count-is-a-word"
),
pytest.param(_usage_with([1, 2]), [1, 2], None, id="details-are-a-list"),
pytest.param(_usage_with(None), _ABSENT, None, id="details-are-null"),
pytest.param(_PLAIN_USAGE, _ABSENT, None, id="details-absent"),
),
)
def test_xai_chat_reports_server_side_tool_usage_from_the_provider_reply(
gateway: Gateway,
usage: dict[str, JsonValue],
expected_details: JsonValue,
expected_web_search_requests: int | None,
) -> None:
with wire_server(_peer(usage)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"xai/{_BACKEND}", api_base=f"{wire.url}/v1", api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "synthetic usage request"}]},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["choices"][0]["message"]["content"] == _REPLY_TEXT, response.text
reported: Final = payload["usage"]
assert isinstance(reported, dict), response.text
assert {name: reported[name] for name in _PLAIN_USAGE} == _PLAIN_USAGE, response.text
assert reported.get("server_side_tool_usage_details", _ABSENT) == expected_details, response.text
prompt_details: Final = reported.get("prompt_tokens_details")
web_search_requests: Final = (
prompt_details.get("web_search_requests") if isinstance(prompt_details, dict) else None
)
assert web_search_requests == expected_web_search_requests, response.text
assert len(wire.drain()) == 1
def test_xai_chat_reports_zero_usage_when_the_provider_usage_is_null(gateway: Gateway) -> None:
with wire_server(_peer(None)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"xai/{_BACKEND}", api_base=f"{wire.url}/v1", api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "synthetic usage request"}]},
)
assert response.status_code == 200, response.text
reported: Final = _JSON_OBJECT.validate_json(response.content)["usage"]
assert isinstance(reported, dict), response.text
assert {name: reported[name] for name in _PLAIN_USAGE} == dict.fromkeys(_PLAIN_USAGE, 0), response.text
assert "server_side_tool_usage_details" not in reported, response.text
assert len(wire.drain()) == 1
@pytest.mark.parametrize("usage", (pytest.param([1, 2], id="usage-is-a-list"), pytest.param("lots", id="usage-is-text")))
def test_xai_chat_answers_500_when_the_provider_usage_is_not_an_object(gateway: Gateway, usage: JsonValue) -> None:
with wire_server(_peer(usage)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"xai/{_BACKEND}", api_base=f"{wire.url}/v1", api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "synthetic usage request"}]},
)
assert response.status_code == 500, response.text
error: Final = _JSON_OBJECT.validate_json(response.content)["error"]
assert isinstance(error, dict), response.text
assert error["type"] == "internal_server_error", response.text
assert "XaiException - Invalid response object" in str(error["message"]), response.text
assert len(wire.drain()) >= 1