The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
use axum::{
body::Bytes,
extract::{FromRequest, FromRequestParts, Multipart, OriginalUri, Request},
http::request::Parts,
response::IntoResponse,
};
use serde_json::{Map, Value};
use crate::{Deployment, Error, Gateway};
pub(crate) const MAX_FILE_BYTES: usize = 50 * 1024 * 1024;
pub(crate) const MAX_BODY_BYTES: usize = MAX_FILE_BYTES + 1024 * 1024;
pub(crate) struct Upload {
pub bytes: Bytes,
pub file_name: Option<String>,
pub mime_type: Option<String>,
}
pub struct JsonObject(pub Map<String, Value>);
pub struct RequestId(pub Option<String>);
impl<S: Send + Sync> FromRequestParts<S> for RequestId {
type Rejection = std::convert::Infallible;
async fn from_request_parts(parts: &mut Parts, _: &S) -> Result<Self, Self::Rejection> {
Ok(Self(
parts
.headers
.get("x-request-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
))
}
}
impl<S: Send + Sync> FromRequest<S> for JsonObject {
type Rejection = Error;
async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> {
let body = Bytes::from_request(request, state).await.map_err(|error| {
if error.status() == axum::http::StatusCode::PAYLOAD_TOO_LARGE {
Error::BodyTooLarge
} else {
Error::InvalidBody(error.to_string())
}
})?;
object(&body).map(Self)
}
}
pub(crate) struct InferenceBody {
pub fields: Map<String, Value>,
pub upload: Option<Upload>,
}
impl<S: Send + Sync> FromRequest<S> for InferenceBody {
type Rejection = Error;
async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> {
let multipart = request
.headers()
.get("content-type")
.and_then(|header| header.to_str().ok())
.is_some_and(|value| {
value
.to_ascii_lowercase()
.starts_with("multipart/form-data")
});
if !multipart {
let JsonObject(fields) = JsonObject::from_request(request, state).await?;
return Ok(Self {
fields,
upload: None,
});
}
let multipart = Multipart::from_request(request, state)
.await
.map_err(|error| Error::InvalidBody(error.to_string()))?;
parse_multipart(multipart).await
}
}
fn object(body: &[u8]) -> Result<Map<String, Value>, Error> {
match serde_json::from_slice(body) {
Ok(Value::Object(body)) => Ok(body),
Ok(_) => Err(Error::InvalidBody("expected a JSON object".into())),
Err(error) => Err(Error::InvalidBody(error.to_string())),
}
}
pub(crate) async fn authorize_model(
identity: &litellm_gateway_auth::AuthenticatedRequest,
deployment: &Deployment,
body: &Map<String, Value>,
) -> Result<(), Error> {
let name = body
.get("model")
.and_then(Value::as_str)
.ok_or_else(|| Error::InvalidBody("model is required".into()))?;
let access = litellm_gateway_auth::AccessRequest::Model {
name: name.into(),
deployment: deployment.model.clone(),
};
identity
.authorize(access.clone())
.await?
.consume(identity.caller(), &access)?;
Ok(())
}
pub(crate) fn resolve_deployment<'a>(
gateway: &'a Gateway,
body: &Map<String, Value>,
) -> Result<&'a Deployment, Error> {
let model = body
.get("model")
.and_then(Value::as_str)
.ok_or_else(|| Error::InvalidBody("model is required".into()))?;
gateway
.models
.get(model)
.ok_or_else(|| Error::UnknownModel(model.to_owned()))
}
async fn parse_multipart(mut multipart: Multipart) -> Result<InferenceBody, Error> {
let mut fields = Map::new();
let mut upload = None;
while let Some(field) = multipart.next_field().await.map_err(multipart_error)? {
let name = field.name().unwrap_or_default().to_owned();
if name == "file" {
let file_name = field.file_name().map(str::to_owned);
let mime_type = field
.content_type()
.and_then(|value| value.split(';').next())
.map(str::trim)
.filter(|value| *value != "application/octet-stream")
.map(str::to_owned);
let bytes = field.bytes().await.map_err(multipart_error)?;
if bytes.len() > MAX_FILE_BYTES {
return Err(Error::BodyTooLarge);
}
upload = Some(Upload {
bytes,
file_name,
mime_type,
});
} else if name != "document" {
let text = field.text().await.map_err(multipart_error)?;
let value = serde_json::from_str(&text).unwrap_or(Value::String(text));
fields.insert(name, value);
}
}
Ok(InferenceBody { fields, upload })
}
fn multipart_error(error: axum::extract::multipart::MultipartError) -> Error {
if error.status() == axum::http::StatusCode::PAYLOAD_TOO_LARGE {
return Error::BodyTooLarge;
}
Error::InvalidBody(error.to_string())
}
pub(crate) async fn unsupported(OriginalUri(uri): OriginalUri) -> impl IntoResponse {
Error::Unsupported(uri.path().to_owned())
}
#[cfg(test)]
mod tests {
use axum::{
Router,
body::{Body, to_bytes},
extract::DefaultBodyLimit,
routing::post,
};
use rstest::{fixture, rstest};
use tower::ServiceExt;
use super::*;
#[fixture]
fn limited_uploads() -> Router {
Router::new()
.route("/", post(|_: InferenceBody| async {}))
.layer(DefaultBodyLimit::max(64))
}
#[rstest]
#[case::json("application/json", format!("{{\"text\":\"{}\"}}", "x".repeat(64)))]
#[case::multipart(
"multipart/form-data; boundary=test",
format!("--test\r\nContent-Disposition: form-data; name=\"file\"; filename=\"file.pdf\"\r\n\r\n{}\r\n--test--\r\n", "x".repeat(64)),
)]
#[tokio::test]
async fn uploads_respect_the_configured_body_limit(
limited_uploads: Router,
#[case] content_type: &str,
#[case] payload: String,
) {
let response = limited_uploads
.oneshot(
Request::post("/")
.header("content-type", content_type)
.body(Body::from(payload))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), 413);
let body: Value =
serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap();
assert_eq!(body["error"]["type"], "request_too_large");
assert_eq!(body["error"]["code"], 413);
}
}