The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import os
import subprocess
import sys
import time
from functools import cache
from pathlib import Path
from typing import Final
IMPORT_SECONDS_VARIABLE: Final = "INTEGRATION_IMPORT_SECONDS"
READINESS_OVERRIDE_VARIABLE: Final = "INTEGRATION_PROXY_READY_SECONDS"
READINESS_IMPORTS: Final = 12
CALL_IMPORTS: Final = 6
PROXY_IMPORT: Final = "import litellm.proxy.proxy_server"
def measure_import_seconds(root: Path, environment: dict[str, str]) -> float:
started: Final = time.monotonic()
subprocess.run(
(sys.executable, "-P", "-c", PROXY_IMPORT), cwd=root, env=environment, capture_output=True, check=True
)
return time.monotonic() - started
def boot_units(workers: int) -> int:
return 1 if workers <= 1 else 2
def readiness_for(imported: float, workers: int) -> float:
return READINESS_IMPORTS * imported * boot_units(workers)
def case_timeout_for(imported: float) -> float:
return readiness_for(imported, 2) + readiness_for(imported, 1) + CALL_IMPORTS * imported
def calibration(imported: float) -> str:
return json.dumps(
{
"import_seconds": imported,
"readiness_seconds_1_worker": readiness_for(imported, 1),
"readiness_seconds_2_workers": readiness_for(imported, 2),
"case_timeout_seconds": case_timeout_for(imported),
},
indent=2,
)
@cache
def import_seconds() -> float:
exported: Final = os.environ.get(IMPORT_SECONDS_VARIABLE)
if exported:
return float(exported)
root: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
return measure_import_seconds(root, dict(os.environ))
def readiness_seconds(workers: int = 1) -> float:
override: Final = os.environ.get(READINESS_OVERRIDE_VARIABLE)
return float(override) if override else readiness_for(import_seconds(), workers)
def case_timeout_seconds() -> float:
return case_timeout_for(import_seconds())