The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
use litellm_host::observation::ObservationSender;
use std::{future::Future, ops::ControlFlow};
use litellm_host::{
call::{HostedCompletion, HostedMachine},
interceptors::Interceptors,
lifecycle::{CallEvent, FailureOrigin, Timing, epoch_seconds},
machine::{Machine, MachineFault},
protocol::Protocol,
};
use crate::{
driver::{Boundary, Driver},
services::HostCallHandler,
};
pub trait StreamConsumer<P: Protocol>: Send + Sync {
fn open_stream(
&self,
head: P::StreamHead,
) -> impl Future<Output = Result<ControlFlow<()>, P::Error>> + Send;
fn send_chunk(
&self,
chunk: P::Chunk,
) -> impl Future<Output = Result<ControlFlow<()>, P::Error>> + Send;
}
impl<P: Protocol> StreamConsumer<P> for () {
async fn open_stream(&self, _: P::StreamHead) -> Result<ControlFlow<()>, P::Error> {
Ok(ControlFlow::Continue(()))
}
async fn send_chunk(&self, _: P::Chunk) -> Result<ControlFlow<()>, P::Error> {
Ok(ControlFlow::Continue(()))
}
}
pub struct Host<'a, S, H, C> {
pub services: &'a S,
pub interceptors: &'a H,
pub stream: &'a C,
pub observers: Option<&'a ObservationSender>,
}
pub async fn run<M, S, H, C>(
machine: M,
host: Host<'_, S, H, C>,
) -> Result<M::Complete, <M::Protocol as Protocol>::Error>
where
M: Machine,
S: HostCallHandler<M::Protocol>,
H: Interceptors<<M::Protocol as Protocol>::Error>,
C: StreamConsumer<M::Protocol>,
{
run_with_completion(machine, host, |_| false).await
}
pub async fn run_hosted<P, S, H, C>(
machine: HostedMachine<P>,
host: Host<'_, S, H, C>,
) -> Result<HostedCompletion<P::Response>, P::Error>
where
P: Protocol,
P::Error: From<MachineFault>,
S: HostCallHandler<P>,
H: Interceptors<P::Error>,
C: StreamConsumer<P>,
{
run_with_completion(machine, host, |completion| {
matches!(completion, HostedCompletion::Detached)
})
.await
}
async fn run_with_completion<M, S, H, C>(
machine: M,
host: Host<'_, S, H, C>,
detached: impl Fn(&M::Complete) -> bool,
) -> Result<M::Complete, <M::Protocol as Protocol>::Error>
where
M: Machine,
S: HostCallHandler<M::Protocol>,
H: Interceptors<<M::Protocol as Protocol>::Error>,
C: StreamConsumer<M::Protocol>,
{
let start_time = epoch_seconds();
if let Some(observers) = host.observers {
observers.emit(CallEvent::Started { start_time });
}
let outcome = consume(
Driver::new(machine, host.services, host.interceptors),
host.stream,
)
.await;
let timing = Timing {
start_time,
end_time: epoch_seconds(),
};
let terminal = match &outcome {
Ok(completion) if detached(completion) => CallEvent::Cancelled { timing },
Ok(_) => CallEvent::Succeeded {
timing,
response: (),
},
Err(_) => CallEvent::Failed {
timing,
origin: FailureOrigin::Call,
error: (),
},
};
if let Some(observers) = host.observers {
observers.emit(terminal);
}
outcome
}
async fn consume<M, S, H, C>(
mut driver: Driver<M, &S, &H>,
stream: &C,
) -> Result<M::Complete, <M::Protocol as Protocol>::Error>
where
M: Machine,
S: HostCallHandler<M::Protocol>,
H: Interceptors<<M::Protocol as Protocol>::Error>,
C: StreamConsumer<M::Protocol>,
{
let mut demand = ControlFlow::Continue(());
loop {
let boundary = match demand {
ControlFlow::Continue(()) => driver.advance().await?,
ControlFlow::Break(()) => driver.detach().await?,
};
let delivered = match boundary {
Boundary::Complete(complete) => return Ok(complete),
Boundary::Open(head) => stream.open_stream(head).await,
Boundary::Chunk(chunk) => stream.send_chunk(chunk).await,
};
demand = match delivered {
Ok(demand) => demand,
Err(error) => return driver.fail(error).await,
};
}
}