The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
use litellm_auth::ResolvedCredential;
use litellm_host_python::{Error as HostError, InvokeError, PythonBinding};
use litellm_host_python::{PythonHostCalls, PythonOwned};
use litellm_inference_ocr::route::{Ocr, OcrCall, OcrOp};
use litellm_llms::base_llm::ocr::error::Error;
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
use pyo3::{
exceptions::{PyBaseException, PyException},
gc::{PyTraverseError, PyVisit},
prelude::*,
types::PyDict,
};
use super::{
errors::to_pyerr as ocr_error_to_pyerr,
project::{OcrHostHandles, SETTINGS_ERROR_MARKER, project_request},
};
use crate::{marshal::public_response, routes::RequestView};
enum OcrHostData {
Unprojected,
Projected(Box<OcrHostHandles>),
Released,
}
/// The Python side of the OCR route: projects the prepared arguments (reading a file-like
/// document as it goes), acquires Azure AD tokens, and builds the public response and
/// exception.
pub(super) struct OcrPythonHost {
view: RequestView,
data: OcrHostData,
}
impl OcrPythonHost {
pub(super) fn new(view: RequestView) -> Self {
Self {
view,
data: OcrHostData::Unprojected,
}
}
fn handles(&self) -> PyResult<&OcrHostHandles> {
match &self.data {
OcrHostData::Projected(handles) => Ok(handles),
_ => Err(HostError::MissingState.into()),
}
}
fn acquire_azure_ad_token(&self, py: Python<'_>) -> PyResult<ResolvedCredential> {
self.handles()?
.azure_ad_token_provider
.as_ref()
.ok_or(HostError::MissingState)?
.acquire(py)
}
fn projection(&mut self, py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult<OcrCall> {
let OcrHostData::Unprojected = self.data else {
return Err(HostError::MissingState.into());
};
self.view.resolve(py, arguments)?;
let (request, handles) = project_request(self.view.request(py)?, arguments)?;
let caller_token = handles.azure_ad_token_provider.is_some();
self.data = OcrHostData::Projected(Box::new(handles));
Ok(OcrCall {
request,
caller_token,
})
}
fn map_failure(&self, py: Python<'_>, error: PyErr) -> PyErr {
if !error.is_instance_of::<PyException>(py) || is_settings_error(py, &error) {
return error;
}
let provider = match &self.data {
OcrHostData::Projected(handles) => handles.provider,
_ => "",
};
let mapped = py
.import("litellm.rust_bridge.host.routes.ocr")
.and_then(|module| module.getattr("map_failure"))
.and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider)))
.and_then(|mapped| mapped.extract::<Py<PyBaseException>>().map_err(PyErr::from));
match mapped {
Ok(mapped) => PyErr::from_value(mapped.into_bound(py).into_any()),
Err(_) => error,
}
}
}
fn is_settings_error(py: Python<'_>, error: &PyErr) -> bool {
error
.value(py)
.getattr_opt(SETTINGS_ERROR_MARKER)
.ok()
.flatten()
.is_some()
}
impl PythonBinding for OcrPythonHost {
type Protocol = Ocr;
type Failure = PyErr;
fn decode_request(
&mut self,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> Result<OcrCall, InvokeError<Error>> {
self.projection(py, arguments)
.map_err(|error| InvokeError::Python(self.map_failure(py, error)))
}
fn encode_response(
&mut self,
py: Python<'_>,
response: LiteLLMOcrResponse,
) -> PyResult<Py<PyAny>> {
public_response(py, "litellm.rust_bridge.host.routes.ocr", &response)
}
fn encode_stream_head(
&mut self,
_: Python<'_>,
head: std::convert::Infallible,
) -> PyResult<Py<PyAny>> {
match head {}
}
fn encode_chunk(
&mut self,
_: Python<'_>,
chunk: std::convert::Infallible,
) -> PyResult<Py<PyAny>> {
match chunk {}
}
fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source)
{
return Ok(original);
}
Ok(self.map_failure(py, ocr_error_to_pyerr(error)))
}
fn host_error(error: &PyErr) -> Error {
Error::InvalidRequest(error.to_string())
}
}
impl PythonHostCalls<Ocr> for OcrPythonHost {
fn handle_host_call(&mut self, py: Python<'_>, op: OcrOp) -> Result<(), InvokeError<Error>> {
match op {
OcrOp::AcquireAzureAdToken(reply) => self
.acquire_azure_ad_token(py)
.map(|token| reply.send(token))
.map_err(|error| InvokeError::Python(self.map_failure(py, error))),
}
}
}
impl PythonOwned for OcrPythonHost {
fn close(&mut self, _: Python<'_>) {
self.view.close();
self.data = OcrHostData::Released;
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.view.traverse(visit)?;
if let OcrHostData::Projected(handles) = &self.data
&& let Some(provider) = &handles.azure_ad_token_provider
{
provider.traverse(visit)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::acquired(true)]
#[case::provider_raised(false)]
fn closing_releases_the_token_provider(#[case] succeeds: bool) {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
locals.set_item("succeeds", succeeds).unwrap();
py.run(
c"
import gc
import weakref
class Provider:
def __call__(self):
if succeeds:
return 'caller-token'
raise ValueError('unavailable')
provider = Provider()
reference = weakref.ref(provider)
kwargs = {
'model': 'azure_ai/mistral-ocr-latest',
'custom_llm_provider': None,
'document': {'type': 'document_url', 'document_url': 'https://example.com/a.pdf'},
'api_key': None,
'api_base': None,
'extra_headers': None,
'timeout': None,
'azure_ad_token_provider': provider,
}
del provider
",
Some(&locals),
Some(&locals),
)
.unwrap();
let kwargs = locals
.get_item("kwargs")
.unwrap()
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let empty = PyDict::new(py);
let mut host = OcrPythonHost::new(RequestView::new(&empty, &empty).unwrap());
assert!(host.decode_request(py, &kwargs).unwrap().caller_token);
locals.del_item("kwargs").unwrap();
drop(kwargs);
let (reply, _) = litellm_host::protocol::reply();
assert_eq!(
host.handle_host_call(py, OcrOp::AcquireAzureAdToken(reply))
.is_ok(),
succeeds
);
let alive = || {
py.run(c"gc.collect()", Some(&locals), Some(&locals))
.unwrap();
!py.eval(c"reference()", Some(&locals), Some(&locals))
.unwrap()
.is_none()
};
assert!(alive());
host.close(py);
assert!(!alive());
});
}
}