The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import itertools
import threading
import uuid
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from typing import Final
import httpx
import pytest
from pydantic import BaseModel, ConfigDict, JsonValue
from tests.integration._support.client import Gateway, eventually, held, string_value
from tests.integration._support.database import read_rows
IN_FLIGHT_CALLERS_PER_PROXY: Final = 8
BLOCK_CYCLES: Final = 4
PROBES_PER_PROXY: Final = 6
class _ObservedRequest(BaseModel):
body: dict[str, JsonValue]
class _Observations(BaseModel):
requests: list[_ObservedRequest]
class _TeamState(BaseModel):
team_id: str
blocked: bool
class _TeamInfo(BaseModel):
team_info: _TeamState
class _ProxyError(BaseModel):
type: str
class _ErrorResponse(BaseModel):
error: _ProxyError
class _Message(BaseModel):
model_config = ConfigDict(frozen=True)
content: str
class _ChatBody(BaseModel):
model_config = ConfigDict(frozen=True)
messages: tuple[_Message, ...]
@pytest.fixture
def upstream(gateway: Gateway) -> Iterator[httpx.Client]:
with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client:
client.delete("/__observations").raise_for_status()
yield client
def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response:
return proxy.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": text}]},
key=key,
)
def _models(client: httpx.Client) -> tuple[str, ...]:
response: Final = client.get("/__observations")
assert response.status_code == 200, response.text
observations: Final = _Observations.model_validate_json(response.text)
return tuple(string_value(request.body["model"]) for request in observations.requests)
def _texts(client: httpx.Client) -> tuple[str, ...]:
response: Final = client.get("/__observations")
assert response.status_code == 200, response.text
observations: Final = _Observations.model_validate_json(response.text)
return tuple(
message.content
for message in itertools.chain.from_iterable(
_ChatBody.model_validate(request.body).messages for request in observations.requests
)
)
def _keep_calling(proxy: Gateway, model: str, key: str, stop: threading.Event) -> tuple[int, ...]:
return tuple(
_chat(proxy, model, key, f"in flight {uuid.uuid4().hex}").status_code
for _ in itertools.takewhile(lambda _: not stop.is_set(), itertools.count())
)
def _statuses_on_fresh_connections(proxies: tuple[Gateway, ...], model: str, key: str, marker: str) -> tuple[int, ...]:
return tuple(
proxy.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": f"{marker} {uuid.uuid4().hex}"}]},
key=key,
headers={"connection": "close"},
).status_code
for proxy in proxies * PROBES_PER_PROXY
)
def _block_while_team_traffic_is_in_flight(
gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str
) -> tuple[httpx.Response, tuple[int, ...]]:
stop: Final = threading.Event()
with ThreadPoolExecutor(max_workers=2 * IN_FLIGHT_CALLERS_PER_PROXY) as pool:
callers: Final = tuple(
pool.submit(_keep_calling, proxy, model, key, stop)
for proxy in (gateway, peer) * IN_FLIGHT_CALLERS_PER_PROXY
)
try:
eventually(
lambda: len(_models(upstream)), lambda served: served >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, seconds=30
)
blocked: Final = gateway.request("POST", "/team/block", {"team_id": team})
finally:
stop.set()
statuses: Final = tuple(itertools.chain.from_iterable(caller.result() for caller in callers))
return blocked, statuses
def _block_and_unblock_under_team_traffic(
gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str, cycle: int
) -> None:
upstream.delete("/__observations").raise_for_status()
blocked, in_flight = _block_while_team_traffic_is_in_flight(gateway, peer, upstream, model, key, team)
assert blocked.status_code == 200, blocked.text
assert in_flight.count(200) >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, repr(in_flight)
assert set(in_flight) <= {200, 401}, repr(in_flight)
marker: Final = f"after block {cycle} {uuid.uuid4().hex}"
refused: Final = held(
lambda: _statuses_on_fresh_connections((gateway, peer), model, key, marker),
lambda statuses: set(statuses) == {401},
holding=5,
seconds=15,
)
assert refused == (401,) * (2 * PROBES_PER_PROXY), repr(refused)
leaked: Final = tuple(text for text in _texts(upstream) if text.startswith(marker))
assert leaked == (), repr(leaked)
unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
assert unblocked.status_code == 200, unblocked.text
eventually(
lambda: _statuses_on_fresh_connections((gateway, peer), model, key, f"after unblock {cycle}"),
lambda statuses: set(statuses) == {200},
seconds=15,
)
def test_team_block_and_unblock_update_endpoint_info_and_db_state(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(model=f"openai/{provider_model}")
team: Final = scenario.team(models=[model])
key: Final = scenario.key(team_id=team)
primary_before: Final = _chat(gateway, model, key, "team block primary warm")
assert primary_before.status_code == 200, primary_before.text
peer_before: Final = _chat(peer, model, key, "team block peer warm")
assert peer_before.status_code == 200, peer_before.text
observed_before_block: Final = _models(upstream)
assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
upstream.delete("/__observations").raise_for_status()
blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
assert blocked_response.status_code == 200, blocked_response.text
blocked: Final = _TeamState.model_validate_json(blocked_response.text)
assert blocked.blocked is True, blocked_response.text
info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team})
assert info_response.status_code == 200, info_response.text
info: Final = _TeamInfo.model_validate_json(info_response.text)
assert info.team_info.blocked is True, info_response.text
assert info.team_info.team_id == team, info_response.text
assert read_rows(
'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
(team,),
) == [{"blocked": True}], info_response.text
cold_key: Final = scenario.key(team_id=team)
cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block")
assert cold_denied.status_code == 401, cold_denied.text
assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text
after_cold_denied: Final = _models(upstream)
assert after_cold_denied == (), repr(after_cold_denied)
upstream.delete("/__observations").raise_for_status()
unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
assert unblocked_response.status_code == 200, unblocked_response.text
restored_key: Final = scenario.key(team_id=team)
restored: Final = _chat(gateway, model, restored_key, "new team key after unblock")
assert restored.status_code == 200, restored.text
after_unblock: Final = _models(upstream)
assert after_unblock == (provider_model,), repr(after_unblock)
unblocked: Final = _TeamState.model_validate_json(unblocked_response.text)
assert unblocked.blocked is False, unblocked_response.text
unblocked_info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team})
assert unblocked_info_response.status_code == 200, unblocked_info_response.text
unblocked_info: Final = _TeamInfo.model_validate_json(unblocked_info_response.text)
assert unblocked_info.team_info.blocked is False, unblocked_info_response.text
assert read_rows(
'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
(team,),
) == [{"blocked": False}], unblocked_info_response.text
assert observed_before_block + after_cold_denied + after_unblock == (
provider_model,
provider_model,
provider_model,
), repr((observed_before_block, after_cold_denied, after_unblock))
def test_unblocking_a_team_restores_warmed_keys_on_both_proxies(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(model=f"openai/{provider_model}")
team: Final = scenario.team(models=[model])
key: Final = scenario.key(team_id=team)
primary_before: Final = _chat(gateway, model, key, "team unblock primary warm")
assert primary_before.status_code == 200, primary_before.text
peer_before: Final = _chat(peer, model, key, "team unblock peer warm")
assert peer_before.status_code == 200, peer_before.text
observed_before_block: Final = _models(upstream)
assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
upstream.delete("/__observations").raise_for_status()
blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
assert blocked_response.status_code == 200, blocked_response.text
cold_key: Final = scenario.key(team_id=team)
cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block")
assert cold_denied.status_code == 401, cold_denied.text
assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text
after_cold_denied: Final = _models(upstream)
assert after_cold_denied == (), repr(after_cold_denied)
upstream.delete("/__observations").raise_for_status()
unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
assert unblocked_response.status_code == 200, unblocked_response.text
primary_after: Final = _chat(gateway, model, key, "team unblock primary")
assert primary_after.status_code == 200, primary_after.text
peer_after: Final = _chat(peer, model, key, "team unblock peer")
assert peer_after.status_code == 200, peer_after.text
after_unblock: Final = _models(upstream)
assert after_unblock == (provider_model, provider_model), repr(after_unblock)
def test_warmed_team_keys_are_refused_on_both_proxies_after_block(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(model=f"openai/{provider_model}")
team: Final = scenario.team(models=[model])
key: Final = scenario.key(team_id=team)
primary_before: Final = _chat(gateway, model, key, "team block primary warm")
assert primary_before.status_code == 200, primary_before.text
peer_before: Final = _chat(peer, model, key, "team block peer warm")
assert peer_before.status_code == 200, peer_before.text
observed_before_block: Final = _models(upstream)
assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
upstream.delete("/__observations").raise_for_status()
blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
assert blocked_response.status_code == 200, blocked_response.text
primary_denied: Final = _chat(gateway, model, key, "blocked team primary")
peer_denied: Final = _chat(peer, model, key, "blocked team peer")
after_block: Final = _models(upstream)
assert (primary_denied.status_code, peer_denied.status_code) == (401, 401), (
f"primary={primary_denied.text}; peer={peer_denied.text}"
)
assert _ErrorResponse.model_validate_json(primary_denied.text).error.type == "auth_error", primary_denied.text
assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text
assert after_block == (), repr(after_block)
assert observed_before_block + after_block == (provider_model, provider_model), repr(
(observed_before_block, after_block)
)
def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(model=f"openai/{provider_model}")
team: Final = scenario.team(models=[model])
warm_key: Final = scenario.key(team_id=team)
peer_before: Final = _chat(peer, model, warm_key, "team block peer warm")
assert peer_before.status_code == 200, peer_before.text
observed_before_block: Final = _models(upstream)
assert observed_before_block == (provider_model,), repr(observed_before_block)
upstream.delete("/__observations").raise_for_status()
blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
assert blocked_response.status_code == 200, blocked_response.text
assert read_rows(
'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
(team,),
) == [{"blocked": True}], blocked_response.text
cold_key: Final = scenario.key(team_id=team)
peer_denied: Final = _chat(peer, model, cold_key, "new team key on peer after block")
after_block: Final = _models(upstream)
assert peer_denied.status_code == 401, peer_denied.text
assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text
assert after_block == (), repr(after_block)
def test_a_team_model_alias_still_routes_on_both_proxies_after_block_and_unblock(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(model=f"openai/{provider_model}")
alias: Final = f"alias-{uuid.uuid4().hex}"
team: Final = scenario.team(models=[model], model_aliases={alias: model})
key: Final = scenario.key(team_id=team, models=[model])
warmed: Final = tuple(
_chat(proxy, alias, key, f"team alias warm {uuid.uuid4().hex}").status_code for proxy in (gateway, peer)
)
assert warmed == (200, 200), repr(warmed)
assert _models(upstream) == (provider_model, provider_model)
upstream.delete("/__observations").raise_for_status()
blocked: Final = gateway.request("POST", "/team/block", {"team_id": team})
assert blocked.status_code == 200, blocked.text
unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
assert unblocked.status_code == 200, unblocked.text
after_unblock: Final = _statuses_on_fresh_connections((gateway, peer), alias, key, "team alias after unblock")
assert after_unblock == (200,) * (2 * PROBES_PER_PROXY), repr(after_unblock)
assert _models(upstream) == (provider_model,) * (2 * PROBES_PER_PROXY)
def test_a_block_issued_while_team_traffic_is_in_flight_holds_on_every_worker(
gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
with gateway.scenario() as scenario:
provider_model: Final = f"team-block-{uuid.uuid4().hex}"
model: Final = scenario.model(
model=f"openai/{provider_model}", input_cost_per_token=0.000001, output_cost_per_token=0.000002
)
team: Final = scenario.team(models=[model])
key: Final = scenario.key(team_id=team)
for cycle in range(BLOCK_CYCLES):
_block_and_unblock_under_team_traffic(gateway, peer, upstream, model, key, team, cycle)