The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import uuid
from collections.abc import Callable
from typing import Final
import pytest
from integration._support.client import Gateway, Scenario, string_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
_EXA_KEY: Final = "synthetic-exa-allowlist-key"
_SEARCH_BODY: Final = {"query": "allowlisted search tools only"}
_EXA_BODY: Final = {"query": "allowlisted search tools only", "contents": {"text": True}}
def _exa_peer(request: Request) -> Reply:
assert (request.method, request.target) == ("POST", "/allowed/search"), request.target
assert request.headers["x-api-key"] == _EXA_KEY, request.headers
assert json.loads(request.body) == _EXA_BODY, request.body
return Reply(body=json.dumps({"results": []}).encode())
def _create_tool(gateway: Gateway, scenario: Scenario, name: str, api_base: str) -> str:
created: Final = gateway.post(
"/search_tools",
{
"search_tool": {
"search_tool_name": name,
"litellm_params": {"search_provider": "exa_ai", "api_key": _EXA_KEY, "api_base": api_base},
"search_tool_info": {"description": f"integration tool {name}"},
}
},
)
identity: Final = string_value(created["search_tool_id"])
scenario.cleanups.callback(gateway.request, "DELETE", f"/search_tools/{identity}")
return identity
def _key_scoped_to(scenario: Scenario, allowed: str) -> str:
user: Final = scenario.user(user_role="internal_user")
return scenario.key(user_id=user, object_permission={"search_tools": [allowed]})
def _team_key_scoped_to(scenario: Scenario, allowed: str) -> str:
team: Final = scenario.team(object_permission={"search_tools": [allowed]})
return scenario.key(team_id=team, user_id=scenario.member(team))
def _listed_names(gateway: Gateway, key: str, path: str, field: str) -> list[str]:
response: Final = gateway.request("GET", path, key=key)
assert response.status_code == 200, response.text
entries: Final = response.json()[field]
return sorted(entry["search_tool_name"] for entry in entries)
def _two_tools(gateway: Gateway, scenario: Scenario, wire: Wire) -> tuple[str, str]:
allowed: Final = f"tool-a-{uuid.uuid4().hex}"
denied: Final = f"tool-b-{uuid.uuid4().hex}"
_create_tool(gateway, scenario, allowed, f"{wire.url}/allowed")
_create_tool(gateway, scenario, denied, f"{wire.url}/denied")
return allowed, denied
def _refusal(scope: str, denied: str, allowed: str) -> dict[str, dict[str, str]]:
return {
"error": {
"message": f"{scope} not allowed to access search tool: {denied}. Allowed search tools: ['{allowed}']",
"type": "key_model_access_denied",
"param": "search_tool_name",
"code": "403",
}
}
@pytest.mark.parametrize(
("scope", "make_key"),
[pytest.param("Key", _key_scoped_to, id="key"), pytest.param("Team", _team_key_scoped_to, id="team")],
)
def test_search_tool_allowlist_refuses_other_tools_before_any_upstream_call(
gateway: Gateway, scope: str, make_key: Callable[[Scenario, str], str]
) -> None:
with wire_server(_exa_peer) as wire, gateway.scenario() as scenario:
allowed, denied = _two_tools(gateway, scenario, wire)
key: Final = make_key(scenario, allowed)
refusal: Final = _refusal(scope, denied, allowed)
for path, body in (
(f"/v1/search/{denied}", _SEARCH_BODY),
("/v1/search", {"search_tool_name": denied, **_SEARCH_BODY}),
):
refused = gateway.request("POST", path, body, key=key)
assert (refused.status_code, refused.json()) == (403, refusal), refused.text
assert wire.drain() == ()
served: Final = gateway.request("POST", f"/v1/search/{allowed}", _SEARCH_BODY, key=key)
assert (served.status_code, served.json()) == (
200,
{"object": "search", "results": [], "provider_metadata": {}},
), served.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/allowed/search")]
_SCOPES: Final = (
pytest.param("Key", _key_scoped_to, id="key"),
pytest.param("Team", _team_key_scoped_to, id="team"),
)
@pytest.mark.parametrize(("scope", "make_key"), _SCOPES)
def test_search_tool_allowlist_refuses_the_model_spelling_before_any_upstream_call(
gateway: Gateway, scope: str, make_key: Callable[[Scenario, str], str]
) -> None:
with wire_server(_exa_peer) as wire, gateway.scenario() as scenario:
allowed, denied = _two_tools(gateway, scenario, wire)
key: Final = make_key(scenario, allowed)
refused: Final = gateway.request("POST", "/v1/search", {"model": denied, **_SEARCH_BODY}, key=key)
assert (refused.status_code, refused.json()) == (403, _refusal(scope, denied, allowed)), refused.text
assert wire.drain() == ()
@pytest.mark.parametrize(("scope", "make_key"), _SCOPES)
def test_search_tools_list_shows_a_scoped_key_only_its_allowlisted_tools(
gateway: Gateway, scope: str, make_key: Callable[[Scenario, str], str]
) -> None:
with wire_server(_exa_peer) as wire, gateway.scenario() as scenario:
allowed, denied = _two_tools(gateway, scenario, wire)
key: Final = make_key(scenario, allowed)
listed: Final = gateway.request("GET", "/search_tools/list", key=key)
assert listed.status_code == 200, listed.text
names: Final = sorted(entry["search_tool_name"] for entry in listed.json()["search_tools"])
assert names == [allowed], f"{scope} scoped to {allowed} listed {names}, {denied} must be hidden: {listed.text}"
assert wire.drain() == ()
def test_search_tools_listing_route_hides_tools_outside_the_key_allowlist(gateway: Gateway) -> None:
pytest.skip(
"BUG: GET /v1/search/tools lists every router search tool to a key whose search_tools allowlist excludes them"
)
with wire_server(_exa_peer) as wire, gateway.scenario() as scenario:
allowed, denied = _two_tools(gateway, scenario, wire)
key: Final = _key_scoped_to(scenario, allowed)
for path in ("/v1/search/tools", "/search/tools"):
assert _listed_names(gateway, key, path, "data") == [allowed]
assert wire.drain() == ()
def _search_tool_row(identity: str) -> list[dict[str, JsonValue]]:
return read_rows(
'SELECT search_tool_name, litellm_params, search_tool_info, updated_at::text AS updated_at FROM "LiteLLM_SearchToolsTable" '
"WHERE search_tool_id = %s",
(identity,),
)
def _non_admin_attempts(
existing: str, existing_name: str, attempted_name: str, api_base: str
) -> tuple[tuple[str, str, dict[str, JsonValue] | None], ...]:
attacker_params: Final = {"search_provider": "exa_ai", "api_key": "attacker", "api_base": api_base}
return (
(
"POST",
"/search_tools",
{"search_tool": {"search_tool_name": attempted_name, "litellm_params": attacker_params}},
),
(
"PUT",
f"/search_tools/{existing}",
{"search_tool": {"search_tool_name": existing_name, "litellm_params": attacker_params}},
),
("DELETE", f"/search_tools/{existing}", None),
("POST", "/search_tools/test_connection", {"litellm_params": attacker_params}),
)
def _delete_tools_named(gateway: Gateway, name: str) -> None:
for row in read_rows('SELECT search_tool_id FROM "LiteLLM_SearchToolsTable" WHERE search_tool_name = %s', (name,)):
gateway.request("DELETE", f"/search_tools/{string_value(row['search_tool_id'])}")
def _refuse_all(request: Request) -> Reply:
raise AssertionError(f"non-admin search tool management reached the provider: {request.target}")
@pytest.mark.parametrize("role", ["internal_user", "internal_user_viewer"])
def test_non_admin_keys_cannot_create_update_delete_or_probe_search_tools(gateway: Gateway, role: str) -> None:
with wire_server(_refuse_all) as wire, gateway.scenario() as scenario:
existing_name: Final = f"admin-owned-{uuid.uuid4().hex}"
existing: Final = _create_tool(gateway, scenario, existing_name, f"{wire.url}/admin")
before: Final = _search_tool_row(existing)
key: Final = scenario.key(user_id=scenario.user(user_role=role))
attempted_name: Final = f"non-admin-{uuid.uuid4().hex}"
scenario.cleanups.callback(_delete_tools_named, gateway, attempted_name)
for method, path, body in _non_admin_attempts(existing, existing_name, attempted_name, f"{wire.url}/attacker"):
response = gateway.request(method, path, body, key=key)
error = response.json().get("error", {}) if response.status_code in (401, 403) else {}
refusal = (error.get("type"), f"Route={path}. Your role={role}." in str(error.get("message")))
assert refusal == ("auth_error", True), f"{method} {path}: {response.status_code} {response.text}"
assert _search_tool_row(existing) == before
assert (
read_rows(
'SELECT search_tool_id FROM "LiteLLM_SearchToolsTable" WHERE search_tool_name = %s', (attempted_name,)
)
== []
)
assert wire.drain() == ()
def test_internal_user_keeps_listing_search_tools(gateway: Gateway) -> None:
with wire_server(_refuse_all) as wire, gateway.scenario() as scenario:
name: Final = f"admin-owned-{uuid.uuid4().hex}"
_create_tool(gateway, scenario, name, f"{wire.url}/admin")
key: Final = scenario.key(user_id=scenario.user(user_role="internal_user"))
assert name in _listed_names(gateway, key, "/search_tools/list", "search_tools")
assert wire.drain() == ()
@pytest.mark.parametrize("role", ["internal_user", "internal_user_viewer"])
def test_non_admin_search_tool_writes_are_forbidden_with_403(gateway: Gateway, role: str) -> None:
pytest.skip("BUG: non-admin POST/PUT/DELETE /search_tools and test_connection return 401 auth_error instead of 403")
with wire_server(_refuse_all) as wire, gateway.scenario() as scenario:
existing_name: Final = f"admin-owned-{uuid.uuid4().hex}"
existing: Final = _create_tool(gateway, scenario, existing_name, f"{wire.url}/admin")
key: Final = scenario.key(user_id=scenario.user(user_role=role))
attempted_name: Final = f"non-admin-{uuid.uuid4().hex}"
scenario.cleanups.callback(_delete_tools_named, gateway, attempted_name)
for method, path, body in _non_admin_attempts(existing, existing_name, attempted_name, f"{wire.url}/attacker"):
response = gateway.request(method, path, body, key=key)
assert (response.status_code, response.json()["error"]["code"]) == (403, "403"), (
f"{method} {path}: {response.text}"
)
assert wire.drain() == ()
def test_internal_user_viewer_can_list_search_tools(gateway: Gateway) -> None:
pytest.skip("BUG: GET /search_tools/list returns 401 to an internal_user_viewer key")
with wire_server(_refuse_all) as wire, gateway.scenario() as scenario:
name: Final = f"admin-owned-{uuid.uuid4().hex}"
_create_tool(gateway, scenario, name, f"{wire.url}/admin")
key: Final = scenario.key(user_id=scenario.user(user_role="internal_user_viewer"))
assert name in _listed_names(gateway, key, "/search_tools/list", "search_tools")
assert wire.drain() == ()