The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import { isDecisionMode } from "@/lib/decisionModels";
const chatCompletionsExample = (baseUrl: string, model: string): string => `import openai
client = openai.OpenAI(
api_key="your_api_key",
base_url=${JSON.stringify(baseUrl)} # Your LiteLLM Proxy URL
)
response = client.chat.completions.create(
model=${JSON.stringify(model)},
messages=[
{
"role": "user",
"content": "Hello, how are you?"
}
]
)
print(response.choices[0].message.content)`;
const systemOneExample = (baseUrl: string, model: string, keyHeaderName: string): string => `import requests
response = requests.post(
${JSON.stringify(`${baseUrl}/v1/systemone`)}, # Your LiteLLM Proxy URL
headers={${JSON.stringify(keyHeaderName)}: "Bearer your_api_key"},
json={
"model": ${JSON.stringify(model)},
"state": "I was charged twice for my subscription this month.",
"questions": {
"wants_refund": {
"type": "noul",
"instructions": "Is the customer asking for money back?"
}
}
}
)
print(response.json()["answers"])`;
const extractionExample = (baseUrl: string, model: string, keyHeaderName: string): string => `import requests
response = requests.post(
${JSON.stringify(`${baseUrl}/v1/extraction`)}, # Your LiteLLM Proxy URL
headers={${JSON.stringify(keyHeaderName)}: "Bearer your_api_key"},
json={
"model": ${JSON.stringify(model)},
"state": "Invoice #4471 from Acme Tools. Total $174.00, due in 30 days.",
"fields": {
"vendor": {"type": "text", "instructions": "Who sent the invoice?"},
"total": {"type": "number", "unit": "USD"}
}
}
)
print(response.json()["answers"])`;
export const modelUsageExample = (
mode: string | null | undefined,
baseUrl: string,
model: string,
keyHeaderName: string,
): string => {
if (mode === "extraction") {
return extractionExample(baseUrl, model, keyHeaderName);
}
return isDecisionMode(mode)
? systemOneExample(baseUrl, model, keyHeaderName)
: chatCompletionsExample(baseUrl, model);
};