The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
pub(crate) mod audio_transcription;
pub(crate) mod chat_completions;
pub(crate) mod clickhouse_spend;
pub(crate) mod embeddings;
mod inference;
pub(crate) mod messages;
pub(crate) mod ocr;
pub(crate) mod responses;
pub(crate) mod token_counter;
pub(crate) mod traces;
use litellm_callbacks_legacy_python::{LegacyLogging, LoggingOperation, PublicCall};
use litellm_host::{call::HostedCompletion, machine::Machine, protocol::Protocol};
use litellm_host_python::{
CallOptions, HookChain, PythonBinding, PythonCallHooks, PythonHostCalls, effective_py_args,
from_py,
};
use litellm_host_python::{Error as HostError, present};
use pyo3::{
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDict, PyMapping, PyTuple},
};
use serde_json::{Map, Value};
/// The public call as Python bound it: `base` holds the positional arguments by name plus
/// the signature defaults, `kwargs` the caller's keyword dict.
#[derive(FromPyObject)]
pub(crate) struct NativeCall<'py> {
#[pyo3(attribute)]
args: Bound<'py, PyTuple>,
#[pyo3(attribute, from_py_with = mapping_dict)]
kwargs: Bound<'py, PyDict>,
#[pyo3(attribute, from_py_with = mapping_dict)]
base: Bound<'py, PyDict>,
}
impl<'py> NativeCall<'py> {
/// The call before any hook ran, for the reads that admit or decline it.
fn resolved(&self) -> PyResult<Bound<'py, PyDict>> {
effective_py_args(&self.base, &self.kwargs)
}
/// The view a route host reads its arguments from.
fn view(&self) -> PyResult<RequestView> {
RequestView::new(&self.base, &self.kwargs)
}
}
/// The keyword view a route host reads: the signature base under the caller's keywords,
/// laid again once the hooks have rewritten them so a keyword a hook deleted stays deleted
/// instead of falling back to the caller's value.
pub(crate) struct RequestView(Option<RequestDicts>);
struct RequestDicts {
base: Py<PyDict>,
request: Py<PyDict>,
}
impl RequestView {
pub(crate) fn new(base: &Bound<'_, PyDict>, kwargs: &Bound<'_, PyDict>) -> PyResult<Self> {
Ok(Self(Some(RequestDicts {
base: base.clone().unbind(),
request: effective_py_args(base, kwargs)?.unbind(),
})))
}
fn dicts(&self) -> PyResult<&RequestDicts> {
self.0
.as_ref()
.ok_or_else(|| HostError::MissingState.into())
}
pub(crate) fn resolve(
&mut self,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> PyResult<()> {
let dicts = self.0.as_mut().ok_or(HostError::MissingState)?;
dicts.request = effective_py_args(dicts.base.bind(py), arguments)?.unbind();
Ok(())
}
pub(crate) fn request<'py>(&self, py: Python<'py>) -> PyResult<&Bound<'py, PyDict>> {
Ok(self.dicts()?.request.bind(py))
}
pub(crate) fn argument<'py>(
&self,
py: Python<'py>,
arguments: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
present(arguments, self.request(py)?, name)
}
/// The body params `module.body_params` keeps from the keywords the call would send, so
/// each route's Python host decides which params reach the provider.
pub(crate) fn body_params(
&self,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
module: &str,
) -> PyResult<Map<String, Value>> {
let request = effective_py_args(self.request(py)?, arguments)?;
from_py(
&py.import(module)?
.getattr("body_params")?
.call1((request,))?,
)
}
pub(crate) fn close(&mut self) {
self.0 = None;
}
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
let Some(dicts) = &self.0 else {
return Ok(());
};
visit.call(&dicts.base)?;
visit.call(&dicts.request)
}
}
fn mapping_dict<'py>(value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyDict>> {
if let Ok(dict) = value.cast::<PyDict>() {
return Ok(dict.clone());
}
let mapping = value.cast::<PyMapping>()?;
let dict = PyDict::new(value.py());
dict.update(mapping)?;
Ok(dict)
}
fn call_hooks(
py: Python<'_>,
operation: LoggingOperation,
call: &NativeCall<'_>,
asynchronous: bool,
) -> PyResult<(Py<PyDict>, impl PythonCallHooks + use<>)> {
let call = PublicCall::capture(&call.base, &call.args, &call.kwargs)?;
let arguments = call.arguments(py);
Ok((
arguments,
LegacyLogging::new(py, operation, call, asynchronous),
))
}
fn execution_steps(py: Python<'_>) -> PyResult<Bound<'_, PyModule>> {
py.import("litellm.rust_bridge.execution")
}
fn run_public_call<H, M>(
py: Python<'_>,
arguments: Py<PyDict>,
start: impl FnOnce(
Python<'_>,
&Bound<'_, PyDict>,
<H::Protocol as Protocol>::Request,
) -> PyResult<M>
+ Send
+ Sync
+ 'static,
host: H,
hooks: impl PythonCallHooks + 'static,
asynchronous: bool,
) -> PyResult<Py<PyAny>>
where
H: PythonBinding + PythonHostCalls<H::Protocol> + 'static,
M: Machine<Protocol = H::Protocol> + 'static,
M::Complete: Into<HostedCompletion<<H::Protocol as Protocol>::Response>>,
{
litellm_host_python::run_call(
py,
move |py, arguments, request| {
start(py, arguments, request).map(crate::logger::LoggedMachine::new)
},
host,
HookChain::new()
.with(hooks)
.with(crate::preflight::SdkPolicy),
arguments,
CallOptions::new(asynchronous, execution_steps),
)
}
#[cfg(test)]
mod tests {
use pyo3::{
prelude::*,
types::{PyDict, PyList},
};
use rstest::rstest;
fn value_call<'py>(
py: Python<'py>,
payload_name: &str,
payload: &Bound<'py, PyAny>,
kwargs: Option<&Bound<'py, PyDict>>,
) -> Bound<'py, PyAny> {
let fields = PyDict::new(py);
fields.set_item("model", "model").unwrap();
fields.set_item(payload_name, payload).unwrap();
if let Some(kwargs) = kwargs {
fields.update(kwargs.as_mapping()).unwrap();
}
let attributes = PyDict::new(py);
attributes
.set_item("args", pyo3::types::PyTuple::empty(py))
.unwrap();
attributes.set_item("kwargs", &fields).unwrap();
attributes.set_item("base", PyDict::new(py)).unwrap();
py.import("types")
.unwrap()
.getattr("SimpleNamespace")
.unwrap()
.call((), Some(&attributes))
.unwrap()
}
fn dict<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyDict> {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap()
}
#[rstest]
#[case::deleted_keyword_stays_deleted("{'system': None}", "{'system': 'caller'}", "{}", None)]
#[case::deleted_keyword_without_default("{}", "{'system': 'caller'}", "{}", None)]
#[case::rewritten_keyword_wins(
"{'system': None}",
"{'system': 'caller'}",
"{'system': 'hook'}",
Some("hook")
)]
#[case::signature_default_survives("{'system': 'default'}", "{}", "{}", Some("default"))]
fn request_view_reads_the_hook_rewritten_keywords_over_the_base(
#[case] base: &str,
#[case] kwargs: &str,
#[case] prepared: &str,
#[case] expected: Option<&str>,
) {
Python::initialize();
Python::attach(|py| {
let mut view = super::RequestView::new(&dict(py, base), &dict(py, kwargs)).unwrap();
let prepared = dict(py, prepared);
view.resolve(py, &prepared).unwrap();
let value = view
.argument(py, &prepared, "system")
.unwrap()
.map(|value| value.extract::<String>().unwrap());
assert_eq!(value.as_deref(), expected);
});
}
#[rstest]
#[case::sync("transcription")]
#[case::asynchronous("atranscription")]
fn route_arguments_that_fail_to_convert_raise_value_error(#[case] name: &str) {
Python::initialize();
Python::attach(|py| {
let module = crate::native_module(py);
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!(
r#"
class Broken:
def __index__(self):
raise LookupError('conversion failed')
value = Broken()
"#
),
Some(&locals),
Some(&locals),
)
.expect("helper class should define");
let broken = locals
.get_item("value")
.expect("locals should be readable")
.expect("helper value should exist");
let error = module
.getattr(name)
.and_then(|function| function.call1((value_call(py, "audio", &broken, None),)))
.expect_err("route should reject a value it cannot convert");
assert!(
error.is_instance_of::<pyo3::exceptions::PyValueError>(py),
"{name} surfaced {error} instead of ValueError"
);
});
}
#[rstest]
fn sync_and_async_routes_apply_the_same_input_validation() {
Python::initialize();
Python::attach(|py| {
let module = crate::native_module(py);
let invalid_headers = PyList::empty(py);
let kwargs = PyDict::new(py);
kwargs
.set_item("extra_headers", &invalid_headers)
.expect("kwargs should accept extra_headers");
let audio = PyDict::new(py);
let sync_error = module
.getattr("transcription")
.and_then(|function| {
function.call1((value_call(py, "audio", &audio, Some(&kwargs)),))
})
.expect_err("sync route should reject non-dict extra_headers");
let async_error = module
.getattr("atranscription")
.and_then(|function| {
function.call1((value_call(py, "audio", &audio, Some(&kwargs)),))
})
.expect_err("async route should reject non-dict extra_headers");
assert_eq!(
sync_error.to_string(),
"ValueError: extra_headers must be a dict"
);
assert_eq!(async_error.to_string(), sync_error.to_string());
});
}
#[rstest]
fn missing_and_explicit_none_optional_params_share_the_next_error() {
Python::initialize();
Python::attach(|py| {
let module = crate::native_module(py);
let audio = PyDict::new(py);
let transcribe = |optional_params: Option<Bound<'_, PyAny>>| {
let kwargs = PyDict::new(py);
if let Some(optional_params) = optional_params {
kwargs.set_item("optional_params", optional_params).unwrap();
}
module
.getattr("transcription")
.and_then(|function| {
function.call1((value_call(py, "audio", &audio, Some(&kwargs)),))
})
.expect_err("model 'model' has no provider")
.to_string()
};
let omitted = transcribe(None);
assert_eq!(transcribe(Some(py.None().into_bound(py))), omitted);
assert_ne!(omitted, "ValueError: optional_params must be a dict");
assert_eq!(
transcribe(Some(PyList::empty(py).into_any())),
"ValueError: optional_params must be a dict"
);
});
}
#[rstest]
fn route_input_validation_preserves_left_to_right_order() {
Python::initialize();
Python::attach(|py| {
let module = crate::native_module(py);
let invalid = PyList::empty(py);
let headers_kwargs = PyDict::new(py);
headers_kwargs
.set_item("extra_headers", &invalid)
.expect("kwargs should accept extra_headers");
let invalid_payload =
PyModule::new(py, "invalid_payload").expect("invalid payload should be created");
let error = module
.getattr("transcription")
.and_then(|function| {
function.call1((value_call(
py,
"audio",
&invalid_payload,
Some(&headers_kwargs),
),))
})
.expect_err("payload should be validated before headers");
assert!(!error.to_string().contains("extra_headers"));
});
}
}