Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

600 B
Raw Blame
mod buffer;
mod caching;
mod codec;
mod embedding;
mod exact;
mod response;
mod service;

pub use buffer::WriteBuffer;
pub use caching::{
    CacheControls, CacheEntry, CacheKeyContext, CacheKeyField, CacheKeyInput, CacheMode, cache_key,
    get_cache_key, should_use_cache,
};
pub use codec::ResponseCacheCodec;
pub use embedding::PartialHits;
pub use exact::{ConnectionProbe, ExactResponseCache};
pub use response::{ResponseCache, ResponseCacheRequest};

pub use service::{
    CacheOptions, CachePolicy, CacheScope, ResponseCacheConfig, ResponseCacheService,
    ResponseEnvelope, ScopedCache,
};