The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import os
import uuid
from collections.abc import Generator, Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
import yaml
from integration._support.client import JSON_OBJECT, Gateway, JsonValue, eventually, list_value, object_value
from integration._support.database import read_rows, scratch_database
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
CALLBACK: Final = "generic_api"
SIBLING: Final = "langsmith"
LIST_TYPES: Final = MappingProxyType(
{
"success_callback": "success",
"failure_callback": "failure",
"callbacks": "success_and_failure",
}
)
SINK_ENVIRONMENT: Final = MappingProxyType(
{
"GENERIC_LOGGER_ENDPOINT": "http://127.0.0.1:9/unused",
"LANGSMITH_API_KEY": "synthetic-langsmith-key",
"LANGSMITH_BASE_URL": "http://127.0.0.1:9",
}
)
NOT_FOUND: Final = f"Callback '{CALLBACK}' not found in active configuration"
WORKER_SYNC_SECONDS: Final = 40
@dataclass(frozen=True, slots=True)
class Rig:
proxy: Gateway
database_url: str
@contextmanager
def _rig(gateway: Gateway, tmp_path: Path, file_settings: Mapping[str, JsonValue]) -> Generator[Rig]:
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
path: Final = tmp_path / f"callback_delete_{uuid.uuid4().hex}.yaml"
path.write_text(
yaml.safe_dump({**config, "litellm_settings": {**object_value(config["litellm_settings"]), **file_settings}})
)
with (
scratch_database() as database_url,
owned_proxy(
gateway,
tmp_path,
{"DATABASE_URL": database_url, **SINK_ENVIRONMENT},
config=path,
remove_environment=("DATABASE_URL_READ_REPLICA",),
workers=2,
) as proxy,
):
yield Rig(proxy, database_url)
@pytest.fixture
def rig(gateway: Gateway, tmp_path: Path) -> Iterator[Rig]:
with _rig(gateway, tmp_path, {}) as created:
yield created
def _configured(proxy: Gateway) -> tuple[tuple[str, str], ...]:
rows: Final = proxy.get("/get/config/callbacks")["callbacks"]
assert isinstance(rows, list)
return tuple(
sorted(
(str(row["name"]), str(row["type"]))
for row in (object_value(entry) for entry in rows)
if not row.get("read_only")
)
)
def _listed(proxy: Gateway, expected: tuple[tuple[str, str], ...]) -> None:
eventually(lambda: _configured(proxy), lambda rows: rows == expected, seconds=WORKER_SYNC_SECONDS)
def _stored_settings(database_url: str) -> Mapping[str, JsonValue]:
rows: Final = read_rows(
'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s',
("litellm_settings",),
database_url=database_url,
)
assert len(rows) == 1, rows
value: Final = rows[0]["param_value"]
return object_value(json.loads(value) if isinstance(value, str) else value)
def _configure(
proxy: Gateway, settings: Mapping[str, JsonValue], environment: Mapping[str, JsonValue] | None = None
) -> None:
body: Final[Mapping[str, JsonValue]] = {
"litellm_settings": dict(settings),
"environment_variables": None if environment is None else dict(environment),
}
assert proxy.post("/config/update", body) == {"message": "Config updated successfully"}
def _delete(proxy: Gateway, name: str) -> tuple[int, Mapping[str, JsonValue]]:
response: Final = proxy.request("POST", "/config/callback/delete", {"callback_name": name})
return response.status_code, JSON_OBJECT.validate_json(response.content)
@pytest.mark.parametrize("list_key", tuple(LIST_TYPES))
def test_callback_listed_under_any_settings_list_is_deleted_from_the_listing_and_the_store(
rig: Rig, list_key: str
) -> None:
_configure(rig.proxy, {list_key: [CALLBACK]})
_listed(rig.proxy, ((CALLBACK, LIST_TYPES[list_key]),))
status, body = _delete(rig.proxy, CALLBACK)
assert status == 200, body
assert (body["removed_callback"], body["remaining_callbacks"]) == (CALLBACK, [])
assert _stored_settings(rig.database_url)[list_key] == []
_listed(rig.proxy, ())
def test_callback_in_both_success_and_failure_lists_is_removed_from_both_in_one_delete(rig: Rig) -> None:
_configure(rig.proxy, {"success_callback": [CALLBACK], "failure_callback": [CALLBACK]})
_listed(rig.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
status, body = _delete(rig.proxy, CALLBACK)
assert status == 200, body
stored: Final = _stored_settings(rig.database_url)
assert (stored["success_callback"], stored["failure_callback"]) == ([], [])
_listed(rig.proxy, ())
def test_delete_keeps_other_callbacks_and_reports_what_is_left_in_the_list_it_edited(rig: Rig) -> None:
_configure(rig.proxy, {"callbacks": [SIBLING, CALLBACK], "failure_callback": [SIBLING]})
_listed(rig.proxy, ((CALLBACK, "success_and_failure"), (SIBLING, "failure"), (SIBLING, "success_and_failure")))
status, body = _delete(rig.proxy, CALLBACK)
assert status == 200, body
assert body["remaining_callbacks"] == [SIBLING]
stored: Final = _stored_settings(rig.database_url)
assert (stored["callbacks"], stored["failure_callback"]) == ([SIBLING], [SIBLING])
_listed(rig.proxy, ((SIBLING, "failure"), (SIBLING, "success_and_failure")))
def test_delete_matches_the_callback_name_case_insensitively(rig: Rig) -> None:
_configure(rig.proxy, {"failure_callback": [CALLBACK.upper()], "callbacks": [CALLBACK]})
_listed(rig.proxy, ((CALLBACK.upper(), "failure"), (CALLBACK, "success_and_failure")))
status, body = _delete(rig.proxy, CALLBACK.upper())
assert status == 200, body
assert body["removed_callback"] == CALLBACK
_listed(rig.proxy, ())
def test_delete_of_an_unconfigured_callback_is_a_404_and_changes_nothing(rig: Rig) -> None:
_configure(rig.proxy, {"callbacks": [SIBLING]})
_listed(rig.proxy, ((SIBLING, "success_and_failure"),))
before: Final = _stored_settings(rig.database_url)
status, body = _delete(rig.proxy, CALLBACK)
assert (status, body) == (404, {"detail": {"error": NOT_FOUND}})
assert _stored_settings(rig.database_url) == before
_listed(rig.proxy, ((SIBLING, "success_and_failure"),))
@pytest.mark.parametrize(
("file_key", "database_key"), (("failure_callback", "success_callback"), ("success_callback", "failure_callback"))
)
def test_delete_removes_the_database_copy_and_keeps_the_copy_the_config_file_sets(
gateway: Gateway, tmp_path: Path, file_key: str, database_key: str
) -> None:
with _rig(gateway, tmp_path, {file_key: [CALLBACK]}) as file_owned:
_configure(file_owned.proxy, {database_key: [CALLBACK]})
_listed(file_owned.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
status, body = _delete(file_owned.proxy, CALLBACK)
assert status == 200, body
assert body["remaining_callbacks"] == []
assert _stored_settings(file_owned.database_url)[database_key] == []
_listed(file_owned.proxy, ((CALLBACK, LIST_TYPES[file_key]),))
def test_delete_of_a_callback_only_the_config_file_lists_is_refused_and_names_the_file(
gateway: Gateway, tmp_path: Path
) -> None:
with _rig(gateway, tmp_path, {"callbacks": [CALLBACK]}) as file_owned:
_listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
status, body = _delete(file_owned.proxy, CALLBACK)
assert status == 400, body
detail: Final = object_value(body["detail"])
assert (detail["keys"], detail["section"]) == (["callbacks"], "litellm_settings")
assert "set in the config file" in str(detail["error"])
_listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
def _completion(request: Request) -> Reply:
message: Final = object_value(list_value(JSON_OBJECT.validate_json(request.body)["messages"])[0])
if str(message["content"]).endswith("-failure"):
return Reply(
status=400,
body=json.dumps(
{"error": {"type": "invalid_request_error", "code": "synthetic", "message": "synthetic failure"}}
).encode(),
)
return Reply(
body=json.dumps(
{
"id": "chatcmpl-" + uuid.uuid4().hex,
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
).encode()
)
@dataclass(slots=True)
class Sink:
wire: Wire
bodies: list[bytes] # mutable-ok: drain() consumes the queue, later reads keep earlier batches
def received(self, markers: tuple[str, ...]) -> bool:
self.bodies.extend(request.body for request in self.wire.drain())
return all(any(marker.encode() in body for body in self.bodies) for marker in markers)
@dataclass(frozen=True, slots=True)
class Pair:
first: Gateway
peer: Gateway
database_url: str
provider: Wire
generic: Sink
langsmith: Sink
def running(self, name: str) -> tuple[bool, bool]:
return (_runs(self.first, name), _runs(self.peer, name))
def _runs(proxy: Gateway, name: str) -> bool:
active: Final = list_value(proxy.get("/active/callbacks")["all_litellm_callbacks"])
return any(name in str(entry).lower() for entry in active)
@contextmanager
def _pair(gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[Pair]:
with (
scratch_database() as database_url,
wire_server(_completion) as provider,
wire_server(lambda _: Reply()) as generic,
wire_server(lambda _: Reply()) as langsmith,
):
monkeypatch.setenv("DATABASE_URL", database_url)
environment: Final = {
"DATABASE_URL": database_url,
"DEFAULT_FLUSH_INTERVAL_SECONDS": "1",
"LITELLM_STORE_AUDIT_LOGS": "True",
}
removed: Final = (
"DATABASE_URL_READ_REPLICA",
"GENERIC_LOGGER_ENDPOINT",
"GENERIC_LOGGER_HEADERS",
"LANGSMITH_API_KEY",
"LANGSMITH_BASE_URL",
)
with (
owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as first,
owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as peer,
):
yield Pair(first, peer, database_url, provider, Sink(generic, []), Sink(langsmith, []))
def _send_on_both(pair: Pair, model: str, key: str, marker: str, outcome: str) -> tuple[str, str]:
markers: Final = (f"{marker}-first-{outcome}", f"{marker}-peer-{outcome}")
for proxy, text in zip((pair.first, pair.peer), markers, strict=True):
response = proxy.request(
"POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": text}]}, key=key
)
assert response.status_code == (400 if outcome == "failure" else 200), response.text
return markers
@pytest.mark.parametrize(("list_key", "outcome"), (("failure_callback", "failure"), ("callbacks", "success")))
def test_deleted_callback_stops_receiving_events_on_both_processes(
gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, list_key: str, outcome: str
) -> None:
assert os.environ.get("LITELLM_LICENSE"), "LITELLM_LICENSE must be set: audit logs are an enterprise feature"
marker: Final = "callback-delete-" + uuid.uuid4().hex
with _pair(gateway, tmp_path, monkeypatch) as pair, pair.first.scenario() as scenario:
_configure(
pair.first,
{list_key: [SIBLING, CALLBACK]},
{
"GENERIC_LOGGER_ENDPOINT": pair.generic.wire.url,
"LANGSMITH_API_KEY": "langsmith-key-" + marker,
"LANGSMITH_BASE_URL": pair.langsmith.wire.url,
},
)
eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
model: Final = scenario.model(api_base=pair.provider.url + "/v1")
key: Final = scenario.key(models=[model])
before: Final = _send_on_both(pair, model, key, marker + "-before", outcome)
eventually(lambda: pair.generic.received(before), bool, seconds=25)
eventually(lambda: pair.langsmith.received(before), bool, seconds=25)
status, body = _delete(pair.first, SIBLING)
assert status == 200, body
assert (body["removed_callback"], body["remaining_callbacks"]) == (SIBLING, [CALLBACK])
assert _stored_settings(pair.database_url)[list_key] == [CALLBACK]
audit: Final = eventually(
lambda: read_rows(
'SELECT before_value, updated_values FROM "LiteLLM_AuditLog" WHERE object_id = %s AND action = %s',
("litellm_settings", "deleted"),
database_url=pair.database_url,
),
bool,
seconds=10,
)
assert len(audit) == 1, audit
before_value: Final = list_value(object_value(audit[0]["before_value"])[list_key])
assert sorted(str(name) for name in before_value) == sorted([SIBLING, CALLBACK]), audit
assert audit[0]["updated_values"] == {list_key: [CALLBACK]}, audit
eventually(
lambda: pair.running(SIBLING), lambda running: running == (False, False), seconds=WORKER_SYNC_SECONDS
)
after: Final = _send_on_both(pair, model, key, marker + "-after", outcome)
eventually(lambda: pair.generic.received(after), bool, seconds=25)
_configure(pair.first, {list_key: [CALLBACK, SIBLING]})
eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
barrier: Final = _send_on_both(pair, model, key, marker + "-barrier", outcome)
eventually(lambda: pair.langsmith.received(barrier), bool, seconds=25)
leaked: Final = tuple(text for text in after if any(text.encode() in body for body in pair.langsmith.bodies))
assert leaked == (), leaked