Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

13.1 KB
Raw Blame
use std::sync::Arc;

use bytes::{Bytes, BytesMut};
use futures_util::future::BoxFuture;
use litellm_auth::AuthServices;
use litellm_host::interceptors::WireRequest;
use litellm_http::{
    Client, ClientVariant, HttpClientConfig, HttpClientPool,
    media::{MediaFetcher, UrlPolicy},
    outbound::{OutboundRequest, RequestSigner},
    transport,
};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;

use crate::base_llm::ocr::{
    error::Error,
    settings::OcrSettings,
    transformation::{
        BaseOcrConfig, DecodedOcrResponse, OcrResponseContext, PreparedOcrRequest,
        decode_request_value, decode_response,
    },
};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument};
use litellm_secrets::source::SecretSource;

/// The route's view of one call, handed to provider code that has to reach the
/// caller's hooks mid-flight (guardrails on the outgoing body, raw response events).
pub trait CallHooks<E>: Send + Sync {
    fn before_provider_request(&self, wire: WireRequest) -> BoxFuture<'_, Result<WireRequest, E>>;

    fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), E>>;
}

#[derive(Clone)]
pub struct OcrClient {
    provider_http: Client,
    polling_http: Client,
    document_fetcher: MediaFetcher,
    auth: Arc<AuthServices>,
    settings: OcrSettings,
    secrets: Arc<dyn SecretSource>,
}

impl OcrClient {
    pub fn new(
        pool: &HttpClientPool,
        config: &HttpClientConfig,
        url_policy: UrlPolicy,
        auth: Arc<AuthServices>,
        settings: OcrSettings,
        secrets: Arc<dyn SecretSource>,
    ) -> Result<Self, litellm_http::Error> {
        Ok(Self {
            provider_http: pool.client(config, ClientVariant::Provider)?,
            polling_http: pool.client(config, ClientVariant::NoRedirect)?,
            document_fetcher: MediaFetcher::new(pool, config, url_policy)?,
            auth,
            settings,
            secrets,
        })
    }

    pub fn provider_http(&self) -> &Client {
        &self.provider_http
    }

    pub fn polling_http(&self) -> &Client {
        &self.polling_http
    }

    pub fn document_fetcher(&self) -> &MediaFetcher {
        &self.document_fetcher
    }

    pub fn auth(&self) -> &AuthServices {
        &self.auth
    }

    pub fn settings(&self) -> &OcrSettings {
        &self.settings
    }

    pub fn secret_source(&self) -> &Arc<dyn SecretSource> {
        &self.secrets
    }

    #[cfg(any(test, feature = "test-support"))]
    pub fn for_test(provider_http: Client, no_redirect_http: Client) -> Self {
        Self {
            secrets: Arc::new(
                litellm_secrets::source::EnvironmentSecrets::python_compatible(
                    provider_http.clone(),
                ),
            ),
            provider_http,
            polling_http: no_redirect_http.clone(),
            document_fetcher: MediaFetcher::for_test(no_redirect_http),
            auth: Arc::new(AuthServices::default()),
            settings: OcrSettings::default(),
        }
    }

    #[cfg(any(test, feature = "test-support"))]
    pub fn with_settings(self, settings: OcrSettings) -> Self {
        Self { settings, ..self }
    }

    #[cfg(any(test, feature = "test-support"))]
    pub fn with_secrets(self, secrets: Arc<dyn SecretSource>) -> Self {
        Self { secrets, ..self }
    }
}

/// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request,
/// send it, and hand the response to the config for normalization.
pub async fn ocr<C: BaseOcrConfig>(
    config: &C,
    client: &OcrClient,
    request: &PreparedOcrRequest,
    hooks: &dyn CallHooks<Error>,
) -> Result<LiteLLMOcrResponse, Error> {
    let http = config.prepare_request(request, client, hooks).await?;
    let url = http.url().to_string();
    let headers = http.headers().to_vec();
    let response = http
        .send(client.provider_http())
        .await
        .map_err(transport_error)?;
    if !response.status().is_success() {
        let headers = response
            .headers()
            .iter()
            .filter_map(|(name, value)| {
                value
                    .to_str()
                    .ok()
                    .map(|value| (name.to_string(), value.to_string()))
            })
            .collect();
        return match read_response_bytes(response, request.connection.max_response_bytes).await {
            Err(Error::Transport(transport::Error::Http { status, body })) => {
                Err(config.get_error_class(body, status, headers))
            }
            Err(error) => Err(error),
            Ok(_) => unreachable!("non-success response produces an HTTP error"),
        };
    }
    let context = OcrResponseContext {
        client,
        connection: &request.connection,
        hooks,
        request_format: request.response_format()?,
        url: &url,
        headers: &headers,
    };
    config
        .async_transform_ocr_response(&request.model, response, context)
        .await
}

pub async fn read_json_response<T: DeserializeOwned>(
    response: reqwest::Response,
    native: bool,
    max_response_bytes: usize,
) -> Result<DecodedOcrResponse<T>, Error> {
    let bytes = read_response_bytes(response, max_response_bytes).await?;
    decode_response(&bytes, native)
}

pub async fn read_response_bytes(
    mut response: reqwest::Response,
    limit: usize,
) -> Result<Bytes, Error> {
    let status = response.status();
    if status.is_success()
        && response
            .content_length()
            .is_some_and(|length| length > limit as u64)
    {
        return Err(Error::TooLarge { limit });
    }
    let mut bytes = BytesMut::new();
    while let Some(chunk) = response.chunk().await.map_err(transport_error)? {
        let remaining = limit.saturating_sub(bytes.len());
        if status.is_success() && chunk.len() > remaining {
            return Err(Error::TooLarge { limit });
        }
        bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
        if !status.is_success() && bytes.len() == limit {
            break;
        }
    }
    if !status.is_success() {
        return Err(transport::Error::Http {
            status: status.as_u16(),
            body: String::from_utf8_lossy(&bytes).into_owned(),
        }
        .into());
    }
    Ok(bytes.freeze())
}

pub fn transport_error(error: reqwest::Error) -> Error {
    if error.is_timeout() {
        return Error::Transport(transport::Error::Http {
            status: 408,
            body: "OCR request timed out".into(),
        });
    }
    transport::Error::from(error).into()
}

pub async fn transform_request_body<C: BaseOcrConfig, B: Serialize>(
    config: &C,
    request: &PreparedOcrRequest,
    url: &str,
    headers: &[(String, String)],
    body: B,
    signer: Option<&dyn RequestSigner>,
    hooks: &dyn CallHooks<Error>,
) -> Result<OutboundRequest, Error> {
    let composed = litellm_core_utils::call_arguments::compose_body(
        &request.optional_params,
        &body,
        config.get_supported_ocr_params(&request.model),
    )?;
    config.validate_request_body(&composed)?;
    let changed = hooks
        .before_provider_request(wire_request(url, headers, composed))
        .await?;
    if !changed.body.is_object() {
        return Err(Error::RequestField {
            path: "guardrail.body".into(),
        });
    }
    config.validate_request_body(&changed.body)?;
    let timeout = Some(request.connection.timeout);
    Ok(match signer {
        Some(signer) => OutboundRequest::signed_json(
            url.into(),
            changed.headers,
            &changed.body,
            timeout,
            signer,
        ),
        None => OutboundRequest::json(url.into(), changed.headers, &changed.body, timeout),
    }?)
}

fn wire_request(url: &str, headers: &[(String, String)], body: Value) -> WireRequest {
    WireRequest {
        url: url.into(),
        headers: headers.to_vec(),
        body,
    }
}

pub fn build_http_request(
    request: &PreparedOcrRequest,
    url: String,
    headers: Vec<(String, String)>,
    body: &impl Serialize,
) -> Result<OutboundRequest, Error> {
    Ok(OutboundRequest::json(
        url,
        headers,
        body,
        Some(request.connection.timeout),
    )?)
}

pub async fn guardrail_document(
    request: &PreparedOcrRequest,
    url: &str,
    headers: &[(String, String)],
    hooks: &dyn CallHooks<Error>,
) -> Result<(OcrDocument, Vec<(String, String)>), Error> {
    let body = serde_json::to_value(&request.document).map_err(|_| Error::RequestField {
        path: "document".into(),
    })?;
    let changed = hooks
        .before_provider_request(wire_request(url, headers, body))
        .await?;
    let document = decode_request_value(changed.body, "guardrail.document")?;
    Ok((document, changed.headers))
}

pub fn body_document(body: &Value) -> Result<OcrDocument, Error> {
    let document = body
        .get("document")
        .and_then(Value::as_object)
        .ok_or_else(|| Error::RequestField {
            path: "body.document".into(),
        })?;
    let source = document
        .iter()
        .filter(|(name, _)| matches!(name.as_str(), "type" | "image_url" | "document_url"))
        .map(|(name, value)| (name.clone(), value.clone()))
        .collect();
    decode_request_value(Value::Object(source), "body.document")
}

#[cfg(test)]
mod tests {
    use crate::base_llm::ocr::{error::Error, handler::read_response_bytes};
    use rstest::rstest;
    use std::time::Duration;
    use tokio::{
        io::{AsyncReadExt, AsyncWriteExt},
        net::TcpListener,
    };

    use super::*;

    #[rstest::rstest]
    #[tokio::test]
    async fn request_timeout_has_an_http_408_status() {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let server = tokio::spawn(async move {
            let _connection = listener.accept().await.unwrap();
            tokio::time::sleep(Duration::from_secs(1)).await;
        });
        let error = litellm_http::Client::plain_for_test()
            .get(format!("http://{address}"))
            .timeout(Duration::from_millis(10))
            .send()
            .await
            .unwrap_err();
        assert!(matches!(
            transport_error(error),
            Error::Transport(transport::Error::Http { status: 408, .. })
        ));
        server.abort();
    }

    /// Answers one request with raw `response` bytes and then holds the connection open, so a
    /// read that waits for the rest of an oversized body hangs instead of passing.
    async fn read_bounded(response: String, limit: usize) -> Result<bytes::Bytes, Error> {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let server = tokio::spawn(async move {
            let (mut socket, _) = listener.accept().await.unwrap();
            let mut request = [0; 4096];
            assert!(socket.read(&mut request).await.unwrap() > 0);
            socket.write_all(response.as_bytes()).await.unwrap();
            std::future::pending::<()>().await;
        });
        let response = litellm_http::Client::plain_for_test()
            .get(format!("http://{address}"))
            .send()
            .await
            .unwrap();
        let result =
            tokio::time::timeout(Duration::from_secs(2), read_response_bytes(response, limit))
                .await;
        server.abort();
        result.expect("bounded reads must finish without waiting for the rest of an oversized body")
    }

    #[rstest]
    #[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh")]
    #[case::chunked(
        "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n4\r\nefgh\r\n0\r\n\r\n"
    )]
    #[tokio::test]
    async fn a_body_of_exactly_the_limit_is_read(#[case] response: &str) {
        assert_eq!(read_bounded(response.into(), 8).await.unwrap(), "abcdefgh");
    }

    #[rstest]
    #[case::declared("HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\n")]
    #[case::chunked(
        "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n5\r\nefghi\r\n"
    )]
    #[tokio::test]
    async fn a_body_over_the_limit_is_rejected(#[case] response: &str) {
        assert!(matches!(
            read_bounded(response.into(), 8).await,
            Err(Error::TooLarge { limit: 8 })
        ));
    }

    #[rstest]
    #[case::declared("Content-Length: 1000000")]
    #[case::chunked("Transfer-Encoding: chunked")]
    #[tokio::test]
    async fn an_oversized_error_keeps_its_status_and_a_bounded_body_without_draining(
        #[case] headers: &str,
    ) {
        let prefix = "x".repeat(4096);
        let body = match headers.starts_with("Transfer") {
            true => format!("{:x}\r\n{prefix}\r\n", prefix.len()),
            false => prefix.clone(),
        };

        let error = read_bounded(
            format!("HTTP/1.1 429 Too Many Requests\r\n{headers}\r\n\r\n{body}"),
            prefix.len(),
        )
        .await
        .unwrap_err();

        let Error::Transport(litellm_http::transport::Error::Http { status, body }) = error else {
            panic!("unexpected error: {error}");
        };
        assert_eq!(status, 429);
        assert_eq!(body, prefix);
    }
}