Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

655 B
Raw Blame
import os

from openai import OpenAI

client = OpenAI(
    base_url="http://0.0.0.0:4000",
    api_key=os.environ["LITELLM_MASTER_KEY"],
)

BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"

# Upload file
batch_input_file = client.files.create(
    file=open("./bedrock_batch_completions.jsonl", "rb"),
    purpose="batch",
    extra_body={"target_model_names": BEDROCK_BATCH_MODEL},
)
print(batch_input_file)

# Create batch
batch = client.batches.create(
    input_file_id=batch_input_file.id,
    endpoint="/v1/chat/completions",
    completion_window="24h",
    metadata={"description": "Test batch job"},
)
print(batch)