The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
mod auth;
mod error;
mod secrets;
pub use error::Error;
use std::{sync::Arc, time::Instant};
use axum::{
Router,
body::{Body, Bytes},
extract::Request,
middleware::Next,
response::Response,
};
use http_body_util::BodyExt;
use litellm_config::Config;
use litellm_gateway_auth::Auth;
use litellm_gateway_inference::{Gateway, ModelRouter};
use litellm_http::{
ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify,
media::PublicDnsResolver,
};
use litellm_inference::resources::CoreResources;
use litellm_secrets::source::EnvironmentSecrets;
use litellm_tracing::ByteChunk;
use uuid::Uuid;
pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
let environment = secrets::environment_values(&config.environment_variables)?;
let lookup = |name: &str| {
environment
.get(name)
.map(|value| value.expose().to_owned())
.or_else(|| std::env::var(name).ok())
};
let settings = &config.litellm_settings;
let http_settings = HttpSettings::from_layers([
HttpSettingsLayer::from_environment(&lookup),
HttpSettingsLayer {
ssl_verify: settings.ssl_verify.as_ref().map(|value| match value {
litellm_config::Spelled::Value(true) => SslVerify::Enabled,
litellm_config::Spelled::Value(false) => SslVerify::Disabled,
litellm_config::Spelled::Text(value) => SslVerify::parse(value),
}),
ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into),
ssl_security_level: settings.ssl_security_level.clone(),
ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(),
force_ipv4: settings.force_ipv4,
http2: settings.http2,
aiohttp_trust_env: settings.aiohttp_trust_env,
disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env,
disable_aiohttp_transport: settings.disable_aiohttp_transport,
..Default::default()
},
]);
let http = Resolution::from(&http_settings).config;
let client = pool.client(&http, ClientVariant::Provider)?;
let secrets = Arc::new(secrets::ConfigSecrets::new(
environment,
Arc::new(EnvironmentSecrets::python_compatible(client)),
));
let resources = CoreResources::new(pool);
Ok(Arc::new(Gateway::new(
resources,
http,
secrets,
ModelRouter::from_model_list(&config.model_list),
)?))
}
pub async fn build_mcp(
config: &Config,
secrets: Arc<dyn litellm_secrets::source::SecretSource>,
shutdown: tokio_util::sync::CancellationToken,
pool: &HttpClientPool,
http: &litellm_http::HttpClientConfig,
) -> Result<Option<litellm_gateway_mcp::ConfiguredGateway>, Error> {
if !config.mcp_tools.is_empty() {
return Err(Error::McpSetting("mcp_tools".into()));
}
if config.mcp_servers.is_empty() {
return Ok(None);
}
if let Some(setting) = config
.general_settings
.additional_fields
.keys()
.chain(config.litellm_settings.additional_fields.keys())
.find(|key| key.starts_with("mcp_"))
{
return Err(Error::McpSetting(setting.clone()));
}
if !config.guardrails.is_empty()
|| !config.policies.is_empty()
|| !config.policy_attachments.is_empty()
{
return Err(Error::McpSetting("guardrails or policies".into()));
}
Auth::from_config(config, secrets.clone())
.validate()
.await?;
let client = pool.mcp_client(http)?;
Ok(Some(
litellm_gateway_mcp::ConfiguredGateway::connect(
&config.mcp_servers,
client,
secrets.as_ref(),
shutdown,
)
.await?,
))
}
pub fn router(
inference: Arc<Gateway>,
config: &Config,
ui: Option<Router>,
mcp: Option<Router>,
) -> Router {
let auth = Auth::from_config(config, inference.secrets.clone());
let inference = litellm_gateway_inference::router(inference)
.merge(mcp.unwrap_or_default())
.route_layer(axum::middleware::from_fn(auth::bind_session_owner))
.route_layer(axum::middleware::from_fn_with_state(
auth,
litellm_gateway_auth::authenticate,
))
.layer(axum::middleware::from_fn(log_request));
match ui {
Some(ui) => inference.merge(ui),
None => inference,
}
}
async fn log_request(request: Request, next: Next) -> Response {
let request_id = Uuid::new_v4().to_string();
let log_body_chunks = tracing::enabled!(tracing::Level::DEBUG);
let method = request.method().clone();
let path = request.uri().path().to_owned();
let started = Instant::now();
let request = if log_body_chunks {
request.map(|body| logged_body(body, request_id.clone(), "input"))
} else {
request
};
let response = next.run(request).await;
tracing::info!(
%request_id,
%method,
%path,
status = response.status().as_u16(),
time_to_headers_ms = started.elapsed().as_secs_f64() * 1000.0,
"response headers"
);
if log_body_chunks {
response.map(|body| logged_body(body, request_id, "output"))
} else {
response
}
}
fn logged_body(body: Body, request_id: String, direction: &'static str) -> Body {
Body::new(body.map_frame(move |frame| {
if let Some(data) = frame.data_ref() {
log_chunk(&request_id, direction, data);
}
frame
}))
}
fn log_chunk(request_id: &str, direction: &str, data: &Bytes) {
let chunk = ByteChunk::new(data);
tracing::debug!(request_id, direction, encoding = chunk.encoding(), chunk = %chunk, "body chunk");
}
#[cfg(test)]
mod tests {
use std::{convert::Infallible, sync::mpsc};
use axum::{body::to_bytes, http::StatusCode, routing::post};
use futures_util::stream;
use litellm_tracing::{Logger, Metadata, Record, Sink};
use rstest::rstest;
use serde_json::{Value, json};
use tower::ServiceExt;
use super::*;
struct LogSink(mpsc::Sender<Value>);
impl Sink for LogSink {
fn enabled(&self, _: &Metadata<'_>) -> bool {
true
}
fn emit(&self, record: &Record) {
self.0
.send(json!({"message": record.message, "fields": record.fields}))
.unwrap();
}
}
#[rstest]
#[tokio::test]
async fn logs_each_body_chunk_without_changing_streamed_bytes() {
let app = Router::new()
.route(
"/stream",
post(|_: Bytes| async {
(
StatusCode::OK,
Body::from_stream(stream::iter([
Ok::<_, Infallible>(Bytes::from_static(b"event: first\n\n")),
Ok(Bytes::from_static(b"event: second\n\n")),
])),
)
}),
)
.layer(axum::middleware::from_fn(log_request));
let request_chunks = [
Ok::<_, Infallible>(Bytes::from_static(b"hello")),
Ok(Bytes::from_static(b" world")),
];
let request = Request::post("/stream")
.body(Body::from_stream(stream::iter(request_chunks)))
.unwrap();
let (sender, receiver) = mpsc::channel();
let logger = Logger::new(LogSink(sender));
let output = logger
.instrument(async {
let response = app.oneshot(request).await.unwrap();
to_bytes(response.into_body(), 1024).await.unwrap()
})
.await;
assert_eq!(output, "event: first\n\nevent: second\n\n");
let records: Vec<Value> = receiver.try_iter().collect();
assert_eq!(records.len(), 5);
assert_eq!(records[0]["fields"]["chunk"], "hello");
assert_eq!(records[1]["fields"]["chunk"], " world");
assert_eq!(records[2]["fields"]["status"], 200);
assert_eq!(records[3]["fields"]["chunk"], "event: first\n\n");
assert_eq!(records[4]["fields"]["chunk"], "event: second\n\n");
let request_id = &records[2]["fields"]["request_id"];
assert!(request_id.as_str().is_some());
assert!(
records
.iter()
.all(|record| &record["fields"]["request_id"] == request_id)
);
}
}