The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from __future__ import annotations
import json
import shutil
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from tests.integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment, object_value, string_value
from tests.integration._support.process import owned_proxy
from tests.integration._support.wire import Reply, Request, Wire, wire_server
_ATTACHABLE: Final = "attachable-during-guard"
_WORDS: Final = "custom-words-during-guard"
_HEADER: Final = "x-litellm-applied-guardrails"
@dataclass(frozen=True, slots=True)
class _Rig:
gateway: Gateway
policy: Wire
upstream: Wire
model: str
def _completion(request: Request) -> Reply:
assert request.target == "/v1/chat/completions", request.target
return Reply(
body=json.dumps(
{
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "guarded"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
).encode()
)
def _allow(request: Request) -> Reply:
assert request.target == "/beta/litellm_basic_guardrail_api", request.target
return Reply(body=json.dumps({"action": "NONE"}).encode())
def _config(directory: Path, policy: Wire) -> Path:
shutil.copy(Path("litellm/proxy/example_config_yaml/custom_guardrail.py"), directory / "custom_guardrail.py")
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
configuration["guardrails"] = [
{
"guardrail_name": _ATTACHABLE,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "during_call",
"api_base": policy.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": _WORDS,
"litellm_params": {"guardrail": "custom_guardrail.myCustomGuardrail", "mode": "during_call"},
},
]
path: Final = directory / "guardrail-attachment.yaml"
path.write_text(yaml.safe_dump(configuration))
return path
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("guardrail-attachment")
with (
wire_server(_allow) as policy,
wire_server(_completion) as upstream,
gateway_from_environment() as shared,
owned_proxy(shared, directory, {}, config=_config(directory, policy)) as owned,
owned.scenario() as scenario,
):
yield _Rig(owned, policy, upstream, scenario.model(api_base=f"{upstream.url}/v1", api_key="sk-fixture"))
def _prompt(text: str) -> str:
return f"{text} {uuid.uuid4().hex}"
def _ask(rig: _Rig, key: str | None, prompt: str, guardrails: list[str] | None = None) -> httpx.Response:
body: Final = {"model": rig.model, "messages": [{"role": "user", "content": prompt}]}
return rig.gateway.request(
"POST", "/v1/chat/completions", body if guardrails is None else {**body, "guardrails": guardrails}, key=key
)
def _served_without_guardrail(rig: _Rig, response: httpx.Response, prompt: str) -> None:
assert response.status_code == 200, response.text
assert _HEADER not in response.headers, dict(response.headers)
assert rig.policy.drain() == ()
forwarded: Final = rig.upstream.drain()
assert len(forwarded) == 1 and prompt in forwarded[0].body.decode(), forwarded
def _served_with_attachable(rig: _Rig, response: httpx.Response, prompt: str) -> None:
assert response.status_code == 200, response.text
assert response.headers[_HEADER] == _ATTACHABLE, dict(response.headers)
inspected: Final = rig.policy.drain()
assert len(inspected) == 1 and prompt in inspected[0].body.decode(), inspected
forwarded: Final = rig.upstream.drain()
assert len(forwarded) == 1 and prompt in forwarded[0].body.decode(), forwarded
def test_a_request_with_an_empty_guardrail_list_is_served_without_the_applied_header(rig: _Rig) -> None:
prompt: Final = _prompt("no guardrails")
_served_without_guardrail(rig, _ask(rig, None, prompt, []), prompt)
def test_a_key_carrying_a_guardrail_applies_it_and_a_plain_key_does_not(rig: _Rig) -> None:
with rig.gateway.scenario() as scenario:
plain: Final = scenario.key()
guarded: Final = scenario.key(guardrails=[_ATTACHABLE])
plain_prompt: Final = _prompt("plain key")
_served_without_guardrail(rig, _ask(rig, plain, plain_prompt), plain_prompt)
guarded_prompt: Final = _prompt("guarded key")
_served_with_attachable(rig, _ask(rig, guarded, guarded_prompt), guarded_prompt)
def test_a_team_carrying_a_guardrail_applies_it_to_its_keys_only(rig: _Rig) -> None:
with rig.gateway.scenario() as scenario:
team: Final = scenario.team(guardrails=[_ATTACHABLE])
outside: Final = scenario.key()
member: Final = scenario.key(team_id=team)
outside_prompt: Final = _prompt("outside team")
_served_without_guardrail(rig, _ask(rig, outside, outside_prompt), outside_prompt)
member_prompt: Final = _prompt("team key")
_served_with_attachable(rig, _ask(rig, member, member_prompt), member_prompt)
def test_a_during_call_custom_guardrail_rejects_a_request_naming_the_banned_word(rig: _Rig) -> None:
unguarded_prompt: Final = _prompt("what is litellm")
_served_without_guardrail(rig, _ask(rig, None, unguarded_prompt), unguarded_prompt)
refused: Final = _ask(rig, None, _prompt("what is litellm"), [_WORDS])
rig.upstream.drain()
assert refused.status_code >= 400, refused.text
error: Final = object_value(JSON_OBJECT.validate_json(refused.content)["error"])
assert "Guardrail failed words - `litellm` detected" in string_value(error["message"]), refused.text
assert rig.policy.drain() == ()