The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, gateway_from_environment, object_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
pytestmark: Final = pytest.mark.shares_worker_state
API_KEY: Final = "wbk_eg_synthetic"
APP_ID: Final = "synthetic-app"
DENIED: Final = "synthetic wingback denied marker"
DENIAL: Final = "synthetic wingback denial"
PRE_CALL: Final = "wingback-pre"
POST_CALL: Final = "wingback-post"
def _verdict(request: Request) -> Reply:
texts: Final = json.loads(request.body).get("texts") or []
denied: Final = any(DENIED in text for text in texts)
result: Final = {"action": "BLOCKED", "blocked_reason": DENIAL} if denied else {"action": "NONE"}
return Reply(body=json.dumps(result).encode())
def _echo_provider(request: Request) -> Reply:
prompt: Final = json.loads(request.body)["messages"][-1]["content"]
return Reply(
body=json.dumps(
{
"id": "chatcmpl-" + uuid.uuid4().hex,
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": f"echo {prompt}"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
).encode()
)
def _wingback(name: str, mode: str, policy: str) -> dict[str, object]:
return {
"guardrail_name": name,
"litellm_params": {
"guardrail": "wingback",
"mode": mode,
"default_on": False,
"api_base": policy,
"api_key": API_KEY,
"wingback_app_id": APP_ID,
},
}
@dataclass(frozen=True, slots=True)
class Rig:
proxy: Gateway
policy: Wire
provider: Wire
model: str
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
root: Final = tmp_path_factory.mktemp("wingback")
with (
gateway_from_environment() as gateway,
wire_server(_verdict) as policy,
wire_server(_echo_provider) as provider,
):
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["litellm_settings"]["cache"] = False
config["guardrails"] = [
_wingback(PRE_CALL, "pre_call", policy.url),
_wingback(POST_CALL, "post_call", policy.url),
]
path: Final = root / "wingback.yaml"
path.write_text(yaml.safe_dump(config))
with (
owned_proxy_process(gateway, root, {}, config=path, workers=2) as owned,
owned.gateway.scenario() as scenario,
):
model: Final = scenario.model(
model="openai/gpt-4o-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
)
yield Rig(owned.gateway, policy, provider, model)
def _chat(rig: Rig, guardrail: str, text: str) -> httpx.Response:
rig.policy.drain()
rig.provider.drain()
return rig.proxy.request(
"POST",
"/v1/chat/completions",
{"model": rig.model, "messages": [{"role": "user", "content": text}], "guardrails": [guardrail]},
)
def _assert_wingback_request(request: Request, input_type: str, text: str) -> None:
assert request.target == "/beta/litellm_basic_guardrail_api", request.target
assert request.headers.get("x-api-key") == API_KEY
body: Final = json.loads(request.body)
assert body["input_type"] == input_type, body
assert any(text in sent for sent in body["texts"]), body
assert body["additional_provider_specific_params"] == {"wingback_app_id": APP_ID}, body
def test_pre_call_denial_never_reaches_the_provider(rig: Rig) -> None:
response: Final = _chat(rig, PRE_CALL, f"{DENIED} {uuid.uuid4().hex}")
assert response.status_code == 400 and DENIAL in response.text, response.text
assert rig.provider.drain() == ()
checks: Final = rig.policy.drain()
assert len(checks) == 1, checks
_assert_wingback_request(checks[0], "request", DENIED)
def test_post_call_denial_blocks_the_provider_response(rig: Rig) -> None:
response: Final = _chat(rig, POST_CALL, f"{DENIED} {uuid.uuid4().hex}")
assert response.status_code == 400 and DENIAL in response.text, response.text
assert len(rig.provider.drain()) == 1
checks: Final = rig.policy.drain()
assert len(checks) == 1, checks
_assert_wingback_request(checks[0], "response", f"echo {DENIED}")
def test_allowed_call_is_checked_once_and_logged_once(rig: Rig) -> None:
prompt: Final = "synthetic wingback allowed prompt " + uuid.uuid4().hex
response: Final = _chat(rig, PRE_CALL, prompt)
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == f"echo {prompt}"
assert len(rig.provider.drain()) == 1
assert response.headers["x-litellm-applied-guardrails"] == PRE_CALL, dict(response.headers)
checks: Final = rig.policy.drain()
assert len(checks) == 1, checks
_assert_wingback_request(checks[0], "request", prompt)
rows: Final = eventually(
lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (response.json()["id"],)),
lambda values: len(values) == 1,
seconds=70,
)
entries: Final = object_value(rows[0]["metadata"])["guardrail_information"]
assert isinstance(entries, list) and len(entries) == 1, entries
entry: Final = object_value(entries[0])
assert entry["guardrail_name"] == PRE_CALL, entry
assert entry["guardrail_mode"] == "pre_call", entry
assert entry["guardrail_status"] == "success", entry