The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
use std::sync::Arc;
use bytes::{Bytes, BytesMut};
use futures_util::future::BoxFuture;
use litellm_auth::AuthServices;
use litellm_host::interceptors::WireRequest;
use litellm_http::{
Client, ClientVariant, HttpClientConfig, HttpClientPool,
media::{MediaFetcher, UrlPolicy},
outbound::{OutboundRequest, RequestSigner},
transport,
};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use crate::base_llm::ocr::{
error::Error,
settings::OcrSettings,
transformation::{
BaseOcrConfig, DecodedOcrResponse, OcrResponseContext, PreparedOcrRequest,
decode_request_value, decode_response,
},
};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument};
use litellm_secrets::source::SecretSource;
/// The route's view of one call, handed to provider code that has to reach the
/// caller's hooks mid-flight (guardrails on the outgoing body, raw response events).
pub trait CallHooks<E>: Send + Sync {
fn before_provider_request(&self, wire: WireRequest) -> BoxFuture<'_, Result<WireRequest, E>>;
fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), E>>;
}
#[derive(Clone)]
pub struct OcrClient {
provider_http: Client,
polling_http: Client,
document_fetcher: MediaFetcher,
auth: Arc<AuthServices>,
settings: OcrSettings,
secrets: Arc<dyn SecretSource>,
}
impl OcrClient {
pub fn new(
pool: &HttpClientPool,
config: &HttpClientConfig,
url_policy: UrlPolicy,
auth: Arc<AuthServices>,
settings: OcrSettings,
secrets: Arc<dyn SecretSource>,
) -> Result<Self, litellm_http::Error> {
Ok(Self {
provider_http: pool.client(config, ClientVariant::Provider)?,
polling_http: pool.client(config, ClientVariant::NoRedirect)?,
document_fetcher: MediaFetcher::new(pool, config, url_policy)?,
auth,
settings,
secrets,
})
}
pub fn provider_http(&self) -> &Client {
&self.provider_http
}
pub fn polling_http(&self) -> &Client {
&self.polling_http
}
pub fn document_fetcher(&self) -> &MediaFetcher {
&self.document_fetcher
}
pub fn auth(&self) -> &AuthServices {
&self.auth
}
pub fn settings(&self) -> &OcrSettings {
&self.settings
}
pub fn secret_source(&self) -> &Arc<dyn SecretSource> {
&self.secrets
}
#[cfg(any(test, feature = "test-support"))]
pub fn for_test(provider_http: Client, no_redirect_http: Client) -> Self {
Self {
secrets: Arc::new(
litellm_secrets::source::EnvironmentSecrets::python_compatible(
provider_http.clone(),
),
),
provider_http,
polling_http: no_redirect_http.clone(),
document_fetcher: MediaFetcher::for_test(no_redirect_http),
auth: Arc::new(AuthServices::default()),
settings: OcrSettings::default(),
}
}
#[cfg(any(test, feature = "test-support"))]
pub fn with_settings(self, settings: OcrSettings) -> Self {
Self { settings, ..self }
}
#[cfg(any(test, feature = "test-support"))]
pub fn with_secrets(self, secrets: Arc<dyn SecretSource>) -> Self {
Self { secrets, ..self }
}
}
/// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request,
/// send it, and hand the response to the config for normalization.
pub async fn ocr<C: BaseOcrConfig>(
config: &C,
client: &OcrClient,
request: &PreparedOcrRequest,
hooks: &dyn CallHooks<Error>,
) -> Result<LiteLLMOcrResponse, Error> {
let http = config.prepare_request(request, client, hooks).await?;
let url = http.url().to_string();
let headers = http.headers().to_vec();
let response = http
.send(client.provider_http())
.await
.map_err(transport_error)?;
if !response.status().is_success() {
let headers = response
.headers()
.iter()
.filter_map(|(name, value)| {
value
.to_str()
.ok()
.map(|value| (name.to_string(), value.to_string()))
})
.collect();
return match read_response_bytes(response, request.connection.max_response_bytes).await {
Err(Error::Transport(transport::Error::Http { status, body })) => {
Err(config.get_error_class(body, status, headers))
}
Err(error) => Err(error),
Ok(_) => unreachable!("non-success response produces an HTTP error"),
};
}
let context = OcrResponseContext {
client,
connection: &request.connection,
hooks,
request_format: request.response_format()?,
url: &url,
headers: &headers,
};
config
.async_transform_ocr_response(&request.model, response, context)
.await
}
pub async fn read_json_response<T: DeserializeOwned>(
response: reqwest::Response,
native: bool,
max_response_bytes: usize,
) -> Result<DecodedOcrResponse<T>, Error> {
let bytes = read_response_bytes(response, max_response_bytes).await?;
decode_response(&bytes, native)
}
pub async fn read_response_bytes(
mut response: reqwest::Response,
limit: usize,
) -> Result<Bytes, Error> {
let status = response.status();
if status.is_success()
&& response
.content_length()
.is_some_and(|length| length > limit as u64)
{
return Err(Error::TooLarge { limit });
}
let mut bytes = BytesMut::new();
while let Some(chunk) = response.chunk().await.map_err(transport_error)? {
let remaining = limit.saturating_sub(bytes.len());
if status.is_success() && chunk.len() > remaining {
return Err(Error::TooLarge { limit });
}
bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
if !status.is_success() && bytes.len() == limit {
break;
}
}
if !status.is_success() {
return Err(transport::Error::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
}
.into());
}
Ok(bytes.freeze())
}
pub fn transport_error(error: reqwest::Error) -> Error {
if error.is_timeout() {
return Error::Transport(transport::Error::Http {
status: 408,
body: "OCR request timed out".into(),
});
}
transport::Error::from(error).into()
}
pub async fn transform_request_body<C: BaseOcrConfig, B: Serialize>(
config: &C,
request: &PreparedOcrRequest,
url: &str,
headers: &[(String, String)],
body: B,
signer: Option<&dyn RequestSigner>,
hooks: &dyn CallHooks<Error>,
) -> Result<OutboundRequest, Error> {
let composed = litellm_core_utils::call_arguments::compose_body(
&request.optional_params,
&body,
config.get_supported_ocr_params(&request.model),
)?;
config.validate_request_body(&composed)?;
let changed = hooks
.before_provider_request(wire_request(url, headers, composed))
.await?;
if !changed.body.is_object() {
return Err(Error::RequestField {
path: "guardrail.body".into(),
});
}
config.validate_request_body(&changed.body)?;
let timeout = Some(request.connection.timeout);
Ok(match signer {
Some(signer) => OutboundRequest::signed_json(
url.into(),
changed.headers,
&changed.body,
timeout,
signer,
),
None => OutboundRequest::json(url.into(), changed.headers, &changed.body, timeout),
}?)
}
fn wire_request(url: &str, headers: &[(String, String)], body: Value) -> WireRequest {
WireRequest {
url: url.into(),
headers: headers.to_vec(),
body,
}
}
pub fn build_http_request(
request: &PreparedOcrRequest,
url: String,
headers: Vec<(String, String)>,
body: &impl Serialize,
) -> Result<OutboundRequest, Error> {
Ok(OutboundRequest::json(
url,
headers,
body,
Some(request.connection.timeout),
)?)
}
pub async fn guardrail_document(
request: &PreparedOcrRequest,
url: &str,
headers: &[(String, String)],
hooks: &dyn CallHooks<Error>,
) -> Result<(OcrDocument, Vec<(String, String)>), Error> {
let body = serde_json::to_value(&request.document).map_err(|_| Error::RequestField {
path: "document".into(),
})?;
let changed = hooks
.before_provider_request(wire_request(url, headers, body))
.await?;
let document = decode_request_value(changed.body, "guardrail.document")?;
Ok((document, changed.headers))
}
pub fn body_document(body: &Value) -> Result<OcrDocument, Error> {
let document = body
.get("document")
.and_then(Value::as_object)
.ok_or_else(|| Error::RequestField {
path: "body.document".into(),
})?;
let source = document
.iter()
.filter(|(name, _)| matches!(name.as_str(), "type" | "image_url" | "document_url"))
.map(|(name, value)| (name.clone(), value.clone()))
.collect();
decode_request_value(Value::Object(source), "body.document")
}
#[cfg(test)]
mod tests {
use crate::base_llm::ocr::{error::Error, handler::read_response_bytes};
use rstest::rstest;
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
use super::*;
#[rstest::rstest]
#[tokio::test]
async fn request_timeout_has_an_http_408_status() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let _connection = listener.accept().await.unwrap();
tokio::time::sleep(Duration::from_secs(1)).await;
});
let error = litellm_http::Client::plain_for_test()
.get(format!("http://{address}"))
.timeout(Duration::from_millis(10))
.send()
.await
.unwrap_err();
assert!(matches!(
transport_error(error),
Error::Transport(transport::Error::Http { status: 408, .. })
));
server.abort();
}
/// Answers one request with raw `response` bytes and then holds the connection open, so a
/// read that waits for the rest of an oversized body hangs instead of passing.
async fn read_bounded(response: String, limit: usize) -> Result<bytes::Bytes, Error> {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0; 4096];
assert!(socket.read(&mut request).await.unwrap() > 0);
socket.write_all(response.as_bytes()).await.unwrap();
std::future::pending::<()>().await;
});
let response = litellm_http::Client::plain_for_test()
.get(format!("http://{address}"))
.send()
.await
.unwrap();
let result =
tokio::time::timeout(Duration::from_secs(2), read_response_bytes(response, limit))
.await;
server.abort();
result.expect("bounded reads must finish without waiting for the rest of an oversized body")
}
#[rstest]
#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh")]
#[case::chunked(
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n4\r\nefgh\r\n0\r\n\r\n"
)]
#[tokio::test]
async fn a_body_of_exactly_the_limit_is_read(#[case] response: &str) {
assert_eq!(read_bounded(response.into(), 8).await.unwrap(), "abcdefgh");
}
#[rstest]
#[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\n")]
#[case::chunked(
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n5\r\nefghi\r\n"
)]
#[tokio::test]
async fn a_body_over_the_limit_is_rejected(#[case] response: &str) {
assert!(matches!(
read_bounded(response.into(), 8).await,
Err(Error::TooLarge { limit: 8 })
));
}
#[rstest]
#[case::declared("Content-Length: 1000000")]
#[case::chunked("Transfer-Encoding: chunked")]
#[tokio::test]
async fn an_oversized_error_keeps_its_status_and_a_bounded_body_without_draining(
#[case] headers: &str,
) {
let prefix = "x".repeat(4096);
let body = match headers.starts_with("Transfer") {
true => format!("{:x}\r\n{prefix}\r\n", prefix.len()),
false => prefix.clone(),
};
let error = read_bounded(
format!("HTTP/1.1 429 Too Many Requests\r\n{headers}\r\n\r\n{body}"),
prefix.len(),
)
.await
.unwrap_err();
let Error::Transport(litellm_http::transport::Error::Http { status, body }) = error else {
panic!("unexpected error: {error}");
};
assert_eq!(status, 429);
assert_eq!(body, prefix);
}
}