The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import logging
from collections.abc import Iterator
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock
import pytest
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
VertexPassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamContext,
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
MODEL = "gemini-stream-pricing-probe"
PROMPT_TOKENS = 1000
COMPLETION_TOKENS = 1000
GEMINI_INPUT_RATE = 1e-07
GEMINI_OUTPUT_RATE = 4e-07
VERTEX_INPUT_RATE = 1.5e-07
VERTEX_OUTPUT_RATE = 6e-07
GEMINI_COST = PROMPT_TOKENS * GEMINI_INPUT_RATE + COMPLETION_TOKENS * GEMINI_OUTPUT_RATE
VERTEX_COST = PROMPT_TOKENS * VERTEX_INPUT_RATE + COMPLETION_TOKENS * VERTEX_OUTPUT_RATE
@pytest.fixture(autouse=True)
def divergent_rate_cards(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setitem(
litellm.model_cost,
f"gemini/{MODEL}",
{
"input_cost_per_token": GEMINI_INPUT_RATE,
"output_cost_per_token": GEMINI_OUTPUT_RATE,
"litellm_provider": "gemini",
"mode": "chat",
},
)
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{MODEL}",
{
"input_cost_per_token": VERTEX_INPUT_RATE,
"output_cost_per_token": VERTEX_OUTPUT_RATE,
"litellm_provider": "vertex_ai",
"mode": "chat",
},
)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def _chunks() -> list[str]:
payload = {
"candidates": [
{
"content": {"parts": [{"text": "hi"}], "role": "model"},
"finishReason": "STOP",
"index": 0,
}
],
"usageMetadata": {
"promptTokenCount": PROMPT_TOKENS,
"candidatesTokenCount": COMPLETION_TOKENS,
"totalTokenCount": PROMPT_TOKENS + COMPLETION_TOKENS,
},
"modelVersion": MODEL,
}
return [f"data: {json.dumps(payload)}"]
def _logging_obj() -> LiteLLMLoggingObj:
logging_obj = MagicMock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {}
logging_obj.optional_params = {}
logging_obj.litellm_call_id = "test-call-id"
return logging_obj
@pytest.mark.parametrize(
"endpoint_type, expected_provider, expected_cost",
[
(EndpointType.GEMINI, "gemini", GEMINI_COST),
(EndpointType.VERTEX_AI, "vertex_ai", VERTEX_COST),
],
)
def test_streaming_generate_content_bills_against_the_requested_provider(
endpoint_type, expected_provider, expected_cost
):
logging_obj = _logging_obj()
_, kwargs = PassThroughStreamingHandler._build_passthrough_logging_result(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route="/v1/generateContent",
request_body={},
endpoint_type=endpoint_type,
start_time=datetime.now(),
raw_bytes=[chunk.encode("utf-8") for chunk in _chunks()],
end_time=datetime.now(),
model=MODEL,
)
assert kwargs["response_cost"] == pytest.approx(expected_cost)
assert logging_obj.model_call_details["custom_llm_provider"] == expected_provider
def test_vertex_generate_content_payload_prices_gemini_urls_at_gemini_rates():
logging_obj = _logging_obj()
result = VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route=f"https://generativelanguage.googleapis.com/v1beta/models/{MODEL}:streamGenerateContent",
request_body={},
endpoint_type=EndpointType.VERTEX_AI,
start_time=datetime.now(),
all_chunks=_chunks(),
model=MODEL,
end_time=datetime.now(),
)
assert result["kwargs"]["response_cost"] == pytest.approx(GEMINI_COST)
assert logging_obj.model_call_details["custom_llm_provider"] == "gemini"
def _interrupted_anthropic_stream(model: str, output_text: str) -> list[bytes]:
def sse(event: str, data: dict) -> bytes:
return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()
message_start = {
"type": "message_start",
"message": {
"id": "msg_interrupted",
"type": "message",
"role": "assistant",
"model": model,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 29, "output_tokens": 2},
},
}
block_start = {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}
delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": output_text}}
return [
sse("message_start", message_start),
sse("content_block_start", block_start),
sse("content_block_delta", delta),
]
@pytest.mark.asyncio
async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(caplog):
from unittest.mock import AsyncMock
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
warm_tokenizer,
)
caplog.set_level(logging.WARNING, logger="LiteLLM")
caplog.set_level(logging.WARNING, logger="LiteLLM Proxy")
model = "claude-fable-5"
warm_tokenizer(model)
logging_obj = _logging_obj()
logging_obj.model_call_details = {"model": model, "stream": True}
logging_obj.litellm_params = {}
logging_obj.get_router_model_id.return_value = None
logging_obj.dispatch_success_handlers = AsyncMock()
_, took, lags = await timed_with_loop_lags(
lambda: PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route="/anthropic/v1/messages",
request_body={"model": model, "stream": True},
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
end_time=datetime.now(),
model=model,
)
)
logging_obj.dispatch_success_handlers.assert_awaited_once()
logged_usage = logging_obj.dispatch_success_handlers.await_args.kwargs["result"].usage
assert logged_usage.completion_tokens > 100_000
assert_loop_stayed_free(took, lags)
@pytest.mark.asyncio
async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(caplog):
from unittest.mock import AsyncMock
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
warm_tokenizer,
)
caplog.set_level(logging.WARNING, logger="LiteLLM")
caplog.set_level(logging.WARNING, logger="LiteLLM Proxy")
model = "claude-fable-5"
warm_tokenizer(model)
logging_obj = _logging_obj()
logging_obj.model_call_details = {"model": model, "stream": True}
logging_obj.litellm_params = {}
logging_obj.get_router_model_id.return_value = None
logging_obj.attempt_scoped_copy.return_value = logging_obj
logging_obj.dispatch_failure_handlers = AsyncMock()
logging_obj.dispatch_attempt_failure_handlers = AsyncMock(
side_effect=logging_obj.dispatch_failure_handlers
)
_, took, lags = await timed_with_loop_lags(
lambda: PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
request_body={"model": model, "stream": True},
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
exception=RuntimeError("upstream closed the stream"),
)
)
await GLOBAL_LOGGING_WORKER.flush()
logging_obj.dispatch_failure_handlers.assert_awaited_once()
partial_usage = logging_obj.record_partial_usage_for_failure.call_args.kwargs["usage"]
assert partial_usage.completion_tokens > 100_000
assert_loop_stayed_free(took, lags)
@pytest.mark.asyncio
async def test_interrupted_vertex_stream_bills_the_deployment_custom_price(monkeypatch: pytest.MonkeyPatch) -> None:
deployment_id: Final = "vertex-custom-priced-deployment"
custom_rates: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09}
monkeypatch.setitem(
litellm.model_cost, deployment_id, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
)
logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[{"role": "user", "content": "ping"}],
stream=True,
call_type="pass_through_endpoint",
start_time=datetime.now(),
litellm_call_id="interrupted-stream",
function_id="interrupted-stream",
)
logging_obj.optional_params = {}
logging_obj.litellm_params = {"metadata": {"model_info": {"id": deployment_id, **custom_rates}}}
url_route: Final = (
f"https://aiplatform.googleapis.com/v1/projects/p/locations/us"
f"/publishers/google/models/{MODEL}:streamGenerateContent?alt=sse"
)
await PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
request_body={},
raw_bytes=[chunk.encode("utf-8") for chunk in _chunks()],
exception=RuntimeError("upstream closed the stream"),
stream_context=PassThroughStreamContext(PassThroughEndpointLogging(), url_route, datetime.now()),
)
recorded: Final = logging_obj.model_call_details["response_cost"]
await GLOBAL_LOGGING_WORKER.flush()
assert recorded == pytest.approx(PROMPT_TOKENS * 2e-09 + COMPLETION_TOKENS * 3e-09)