The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from collections.abc import Awaitable, Callable, Mapping
from typing import Final, cast # noqa: TID251 # narrows legacy callable signatures
import pytest
import litellm
from litellm.caching.caching import Cache
from litellm.chat_completions import dispatch
from litellm.chat_completions.dispatch import (
_ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch
_DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.rust_bridge.bindings import NativeBinding
from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules
from litellm.rust_bridge.chat_completions.entrypoints import (
NATIVE_ACOMPLETION,
NATIVE_COMPLETION,
NativeAcompletion,
NativeCompletion,
)
from litellm.rust_bridge.configuration import Rollout
from litellm.rust_bridge.public_call import NativeCall, native_call_hook
from litellm.types.utils import ModelResponse
MESSAGES: Final = [{"role": "user", "content": "hi"}]
PYTHON_RULES: Final = ()
RUST_RULES: Final = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),)
def completion_binding(native: NativeCompletion | None) -> NativeBinding[NativeCompletion]:
binding: Final[NativeBinding[NativeCompletion]] = NativeBinding("completion", validate=lambda _: None)
binding.override(native)
return binding
def acompletion_binding(native: NativeAcompletion | None) -> NativeBinding[NativeAcompletion]:
binding: Final[NativeBinding[NativeAcompletion]] = NativeBinding("acompletion", validate=lambda _: None)
binding.override(native)
return binding
def test_python_route_forwards_original_call_shape() -> None:
metadata: Final = {"user_id": "u"}
args: Final[tuple[object, ...]] = ("gpt-4o", MESSAGES)
kwargs: Final[Mapping[str, object]] = {"temperature": 0.1, "metadata": metadata}
captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = []
response: Final = ModelResponse()
def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: records public call shape
captured.append((call_args, call_kwargs))
return response
def native(request: NativeCall) -> ModelResponse:
pytest.fail("Python-only dispatch must not call native")
assert (
_DISPATCH.run(
args,
kwargs,
python=python,
binding=completion_binding(native),
native=native_call_hook,
rules=PYTHON_RULES,
)
is response
)
call_args, call_kwargs = captured[0]
assert call_args == args
assert call_args[1] is MESSAGES
assert call_kwargs == kwargs
assert call_kwargs["metadata"] is metadata
assert kwargs == {"temperature": 0.1, "metadata": metadata}
@pytest.mark.asyncio
async def test_async_python_route_forwards_original_call_shape() -> None:
metadata: Final = {"user_id": "u"}
args: Final[tuple[object, ...]] = ("gpt-4o", MESSAGES)
kwargs: Final[Mapping[str, object]] = {"temperature": 0.1, "metadata": metadata}
captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = []
response: Final = ModelResponse()
async def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: records call shape
captured.append((call_args, call_kwargs))
return response
async def native(request: NativeCall) -> ModelResponse:
pytest.fail("Python-only dispatch must not call native")
result: Final = await _ADISPATCH.arun(
args,
kwargs,
python=python,
binding=acompletion_binding(native),
native=native_call_hook,
rules=PYTHON_RULES,
)
assert result is response
call_args, call_kwargs = captured[0]
assert call_args == args
assert call_args[1] is MESSAGES
assert call_kwargs == kwargs
assert call_kwargs["metadata"] is metadata
assert kwargs == {"temperature": 0.1, "metadata": metadata}
def test_native_receives_bound_request_and_original_call_shape() -> None:
metadata: Final = {"user_id": "u"}
kwargs: Final[Mapping[str, object]] = {
"stream": True,
"api_key": "sk-test",
"base_url": "https://example.invalid",
"extra_headers": {"x-test": "1"},
"custom_llm_provider": "anthropic",
"metadata": metadata,
}
captured: Final[list[tuple[NativeCall, tuple[object, ...], Mapping[str, object]]]] = []
def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: rejected Rust fallback
pytest.fail("Required Rust dispatch must not call Python")
def native(request: NativeCall) -> ModelResponse:
captured.append((request, request.args, request.kwargs))
return ModelResponse()
args: Final[tuple[object, ...]] = ("anthropic/claude-sonnet-4-5", MESSAGES)
_DISPATCH.run(
args,
kwargs,
python=python,
binding=completion_binding(native),
native=native_call_hook,
rules=RUST_RULES,
)
request, call_args, call_kwargs = captured[0]
assert request.resolved["model"] == "anthropic/claude-sonnet-4-5"
assert request.resolved["messages"] is MESSAGES
assert request.resolved["stream"] is True
assert request.resolved["api_key"] == "sk-test"
assert request.resolved["base_url"] == "https://example.invalid"
assert request.resolved["custom_llm_provider"] == "anthropic"
assert request.resolved["extra_headers"] == {"x-test": "1"}
assert request.kwargs is kwargs
assert call_args == args
assert call_kwargs == kwargs
assert call_kwargs["metadata"] is metadata
@pytest.mark.parametrize(
("args", "kwargs"),
(
(("gpt-4o", MESSAGES), {"model": "duplicate"}),
((), {}),
),
)
def test_binding_errors_delegate_to_python(args: tuple[object, ...], kwargs: Mapping[str, object]) -> None:
captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = []
response: Final = ModelResponse()
def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: records invalid call shape
captured.append((call_args, call_kwargs))
return response
def native(request: NativeCall) -> ModelResponse:
pytest.fail("Binding failures must be delegated to Python")
assert (
_DISPATCH.run(
args,
kwargs,
python=python,
binding=completion_binding(native),
native=native_call_hook,
rules=RUST_RULES,
)
is response
)
assert captured == [(args, kwargs)]
def test_sync_completion_request_projects_public_arguments() -> None:
rules: Final[Rules] = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),)
expected: Final = ModelResponse()
def native(request: NativeCall) -> ModelResponse:
assert request.resolved["model"] == "test-model"
assert request.resolved["messages"] == MESSAGES
assert request.resolved["custom_llm_provider"] == "openai"
assert request.resolved["stream"] is True
return expected
binding: Final[NativeBinding[NativeCompletion]] = NativeBinding("completion", validate=lambda _: None)
binding.override(native)
response: Final = dispatch._DISPATCH.run( # pyright: ignore[reportPrivateUsage] # test an explicit route decision
("test-model", MESSAGES),
{"custom_llm_provider": "openai", "stream": True},
python=lambda *args, **kwargs: pytest.fail("required native route must handle this call"),
binding=binding,
native=native_call_hook,
rules=rules,
)
assert response is expected
def test_positional_parameters_remain_available_to_native_projection() -> None:
request: Final = _DISPATCH.request(("anthropic/test-model", MESSAGES, 12.0, 0.25), {})
assert request is not None
assert request.resolved["timeout"] == 12.0
assert request.resolved["temperature"] == 0.25
assert request.resolved["messages"] is MESSAGES
@pytest.mark.asyncio
async def test_acompletion_marker_binds_positionals_by_the_sync_signature() -> None:
captured: Final[list[NativeCall]] = []
async def native(call: NativeCall) -> ModelResponse:
captured.append(call)
return ModelResponse()
NATIVE_ACOMPLETION.override(native)
public_completion: Final = cast(Callable[..., Awaitable[ModelResponse]], litellm.completion)
try:
_ = await public_completion("gpt-4o", MESSAGES, 30.0, acompletion=True)
finally:
NATIVE_ACOMPLETION.reset()
assert [(request.resolved["timeout"], request.resolved["functions"]) for request in captured] == [(30.0, None)]
class _ChatAgentic(CustomLogger):
async def async_should_run_chat_completion_agentic_loop(
self, *args: object, **kwargs: object
) -> tuple[bool, dict[str, object]]: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override
return False, {}
@pytest.mark.parametrize(
("feature", "kwargs", "settings"),
(
(Feature.MOCK_RESPONSE, {"mock_response": "hi"}, {}),
pytest.param(Feature.MOCK_RESPONSE, {"mock_tool_calls": [{"id": "call_1"}]}, {}, id="mock-tool-calls"),
pytest.param(Feature.MOCK_RESPONSE, {"mock_timeout": True}, {}, id="mock-timeout"),
(Feature.CALLER_CLIENT, {"client": object()}, {}),
(Feature.TIMEOUT_VARIANTS, {"request_timeout": 5}, {}),
(Feature.AGENTIC_LOOP_HOOK, {}, {"callbacks": [_ChatAgentic()]}),
(Feature.TLS_POLICY, {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}),
(Feature.RESPONSE_CACHE, {}, {"cache": Cache()}),
(Feature.STREAMING, {"stream": True}, {}),
(Feature.DROP_PARAMS, {}, {"drop_params": True}),
(Feature.MODIFY_PARAMS, {}, {"modify_params": True}),
(Feature.EXTRA_BODY, {"extra_body": {"top_k": 1}}, {}),
(Feature.RETRIES_FALLBACKS, {"num_retries": 2}, {}),
pytest.param(
Feature.RETRIES_FALLBACKS,
{"context_window_fallback_dict": {"anthropic/claude-sonnet-4-5": "anthropic/claude-opus-4-5"}},
{},
id="context-window-fallbacks",
),
pytest.param(
Feature.RETRIES_FALLBACKS,
{},
{"model_fallbacks": [{"anthropic/claude-sonnet-4-5": ["anthropic/claude-opus-4-5"]}]},
id="model-fallbacks",
),
(Feature.PROMPT_MANAGEMENT, {"prompt_id": "welcome"}, {}),
pytest.param(Feature.PROMPT_MANAGEMENT, {"model": "litellm_agent/gpt-5"}, {}, id="litellm-agent-model"),
(Feature.CACHE_CONTROL_INJECTION, {"enable_prompt_caching": True}, {}),
(Feature.MODEL_PARAM_GATING, {"logit_bias": {"1": 1}}, {}),
(
Feature.MODEL_LIST,
{"model_list": [{"model_name": "anthropic/claude-sonnet-4-5", "litellm_params": {"model": "gpt-5"}}]},
{},
),
(
Feature.MODEL_ALIAS_MAP,
{},
{"model_alias_map": {"anthropic/claude-sonnet-4-5": "anthropic/claude-opus-4-5"}},
),
),
)
def test_each_detected_feature_reaches_its_rule(
monkeypatch: pytest.MonkeyPatch, feature: Feature, kwargs: Mapping[str, object], settings: Mapping[str, object]
) -> None:
rules: Final = (
RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY, features=frozenset({feature})),
RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),
)
python_result: Final = ModelResponse()
native_result: Final = ModelResponse()
def served(call_kwargs: Mapping[str, object]) -> object:
return _DISPATCH.run(
(),
{"model": "anthropic/claude-sonnet-4-5", "messages": MESSAGES, **call_kwargs},
python=lambda *call_args, **python_kwargs: python_result,
binding=completion_binding(lambda request: native_result),
native=native_call_hook,
rules=rules,
)
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "ssl_security_level", None)
monkeypatch.setattr(litellm, "cache", None)
monkeypatch.setattr(litellm, "model_alias_map", {})
for name in ("num_retries", "model_fallbacks"):
monkeypatch.setattr(litellm, name, None)
for name in ("drop_params", "modify_params", "enable_anthropic_prompt_caching"):
monkeypatch.setattr(litellm, name, False)
for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"):
monkeypatch.delenv(name, raising=False)
assert served({}) is native_result
for name, value in settings.items():
monkeypatch.setattr(litellm, name, value)
assert served(kwargs) is python_result
def test_acompletion_flag_hands_off_only_when_true() -> None:
expected: Final = ModelResponse()
def native(call: NativeCall) -> ModelResponse:
return expected
async def anative(call: NativeCall) -> ModelResponse:
pytest.fail("completion(..., acompletion=1) stays sync, since the Python @client only hands off on is True")
NATIVE_COMPLETION.override(native)
NATIVE_ACOMPLETION.override(anative)
public_completion: Final = cast(Callable[..., object], litellm.completion)
try:
result: Final = public_completion("gpt-4o", MESSAGES, acompletion=1)
finally:
NATIVE_COMPLETION.reset()
NATIVE_ACOMPLETION.reset()
assert result is expected