The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import json
from datetime import datetime, timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from pydantic import TypeAdapter
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
EndpointType,
PassthroughStandardLoggingPayload,
)
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
# Helper function to mock async iteration
async def aiter_mock(iterable):
for item in iterable:
yield item
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"endpoint_type,url_route",
[
(
EndpointType.VERTEX_AI,
"v1/projects/pathrise-convert-1606954137718/locations/us-central1/publishers/google/models/gemini-1.0-pro:generateContent",
),
(EndpointType.ANTHROPIC, "/v1/messages"),
],
)
async def test_chunk_processor_yields_raw_bytes(endpoint_type, url_route):
"""
Test that the chunk_processor yields raw bytes
This is CRITICAL for pass throughs streaming with Vertex AI and Anthropic
"""
# Mock inputs
response = AsyncMock(spec=httpx.Response)
response.status_code = 200
raw_chunks = [
b'{"id": "1", "content": "Hello"}',
b'{"id": "2", "content": "World"}',
b'\n\ndata: {"id": "3"}', # Testing different byte formats
]
# Mock aiter_bytes to return an async generator
async def mock_aiter_bytes():
for chunk in raw_chunks:
yield chunk
response.aiter_bytes = mock_aiter_bytes
request_body = {"key": "value"}
litellm_logging_obj = MagicMock()
start_time = datetime.now()
passthrough_success_handler_obj = MagicMock()
litellm_logging_obj.async_success_handler = AsyncMock()
# Capture yielded chunks and perform detailed assertions
received_chunks = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
):
# Assert each chunk is bytes
assert isinstance(chunk, bytes), f"Chunk should be bytes, got {type(chunk)}"
# Assert no decoding/encoding occurred (chunk should be exactly as input)
assert chunk in raw_chunks, (
f"Chunk {chunk} was modified during processing. For pass throughs streaming, chunks should be raw bytes"
)
received_chunks.append(chunk)
# Assert all chunks were processed
assert len(received_chunks) == len(raw_chunks), "Not all chunks were processed"
# collected chunks all together
assert b"".join(received_chunks) == b"".join(raw_chunks), "Collected chunks do not match raw chunks"
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_route_streaming_logging_runs_async_handler_for_sdk_passthrough():
"""
SDK pass-through streaming (anthropic_messages, google generate_content) must run
the async success handler so async-only loggers record the assembled stream.
Regression for duplicate-trace dedupe: dispatch_success_handlers treated these as
sync SDK requests because call_type is not ``pass_through_endpoint`` and
litellm_params carries no ``acompletion`` flag, so only the sync success_handler
ran and CustomLogger.async_log_success_event never fired.
"""
import time
from litellm.types.utils import CallTypes
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type=CallTypes.anthropic_messages.value,
start_time=time.time(),
litellm_call_id="test-id",
function_id="fn",
)
logging_obj.model_call_details["litellm_params"] = {"anthropic_messages": True}
with (
patch.object(
PassThroughStreamingHandler,
"_build_passthrough_logging_result",
return_value=({"id": "slp"}, {}),
),
patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async,
patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_callbacks_for_async_calls",
return_value=False,
),
):
await PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
request_body={},
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
raw_bytes=[],
end_time=datetime.now(),
)
mock_async.assert_awaited_once()
mock_sync.assert_not_called()
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_handle_logging_runs_async_handler_for_passthrough():
"""
Non-streaming pass-through logging (_handle_logging) must always run the
async success handler so async-only loggers (e.g. the proxy spend logger)
record the request.
_handle_logging is only ever reached from pass_through_async_success_handler
(an async context), so it forces async dispatch via prefer_async_handlers.
This pins that contract independent of the call-type classification: even a
call_type that _is_sync_litellm_request would classify as sync (here
"completion" with no async marker in litellm_params) must still reach
async_success_handler. Without prefer_async_handlers=True the sync-only
branch would return early and async_log_success_event would never fire.
"""
import time
from litellm.types.utils import CallTypes
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type=CallTypes.completion.value,
start_time=time.time(),
litellm_call_id="test-id",
function_id="fn",
)
logging_obj.model_call_details["litellm_params"] = {}
handler = PassThroughEndpointLogging()
with (
patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async,
patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_callbacks_for_async_calls",
return_value=False,
),
):
await handler._handle_logging(
logging_obj=logging_obj,
standard_logging_response_object={"id": "slp"},
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
)
mock_async.assert_awaited_once()
mock_sync.assert_not_called()
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_unhandled_route_passes_passthrough_payload_to_custom_logger():
class _PayloadLogger(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payload: PassthroughStandardLoggingPayload | None = None
async def async_log_success_event(
self,
kwargs: dict[str, object],
response_obj: object,
start_time: datetime,
end_time: datetime,
) -> None:
self.payload = TypeAdapter(PassthroughStandardLoggingPayload).validate_python(
kwargs["passthrough_logging_payload"]
)
start_time: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
end_time: Final = datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc)
url: Final = "https://upstream.example/v1/moderations"
request_body: Final = {"model": "omni-moderation-latest", "input": "check this"}
response_body: Final = {"id": "moderation-1", "results": [{"flagged": False}]}
payload: Final = PassthroughStandardLoggingPayload(
url=url,
request_body=request_body,
request_method="POST",
response_body=response_body,
)
logger: Final = _PayloadLogger()
logging_obj: Final = LiteLLMLoggingObj(
model="omni-moderation-latest",
messages=[],
stream=False,
call_type="pass_through_endpoint",
start_time=start_time,
litellm_call_id="moderation-1",
function_id="moderations",
dynamic_async_success_callbacks=[logger],
)
response: Final = httpx.Response(200, request=httpx.Request("POST", url), json=response_body)
await PassThroughEndpointLogging().pass_through_async_success_handler(
httpx_response=response,
response_body=response_body,
logging_obj=logging_obj,
url_route="/v1/moderations",
result="",
start_time=start_time,
end_time=end_time,
cache_hit=False,
request_body=request_body,
passthrough_logging_payload=payload,
)
assert logger.payload == payload
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
def test_convert_raw_bytes_to_str_lines():
"""
Test that the _convert_raw_bytes_to_str_lines method correctly converts raw bytes to a list of strings
"""
# Test case 1: Single chunk
raw_bytes = [b'data: {"content": "Hello"}\n']
result = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
assert result == ['data: {"content": "Hello"}']
# Test case 2: Multiple chunks
raw_bytes = [b'data: {"content": "Hello"}\n', b'data: {"content": "World"}\n']
result = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
assert result == ['data: {"content": "Hello"}', 'data: {"content": "World"}']
# Test case 3: Empty input
raw_bytes = []
result = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
assert result == []
# Test case 4: Chunks with empty lines
raw_bytes = [b'data: {"content": "Hello"}\n\n', b'\ndata: {"content": "World"}\n']
result = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
assert result == ['data: {"content": "Hello"}', 'data: {"content": "World"}']
@pytest.fixture()
async def _drain_logging_worker():
"""
The logging queue is bound to the running loop, so anything left queued when a test's loop
goes away is carried onto the next loop and fires against that test's callbacks.
"""
GLOBAL_LOGGING_WORKER.start()
try:
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10)
except asyncio.TimeoutError:
pass
await GLOBAL_LOGGING_WORKER.stop()
yield
@pytest.fixture()
def _vcr_outcome_gate(request, vcr):
install_live_call_probe(request, vcr)
yield
record_vcr_outcome(request, vcr)
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_vertex_ai_anthropic_streaming_cost_injection_enabled():
"""
Test that cost is injected into Vertex AI streamRawPredict streaming chunks
when include_cost_in_streaming_usage is enabled.
"""
# Enable cost injection
original_value = getattr(litellm, "include_cost_in_streaming_usage", False)
litellm.include_cost_in_streaming_usage = True
try:
# Mock response with Anthropic SSE format chunks
response = AsyncMock(spec=httpx.Response)
response.status_code = 200
# Create chunks with message_delta event containing usage
chunks_with_usage = [
b'data: {"type": "content_block_delta", "delta": {"text": "Hello"}}\n\n',
b'data: {"type": "message_delta", "usage": {"input_tokens": 10, "output_tokens": 5}}\n\n',
b'data: {"type": "content_block_delta", "delta": {"text": " world"}}\n\n',
]
async def mock_aiter_bytes():
for chunk in chunks_with_usage:
yield chunk
response.aiter_bytes = mock_aiter_bytes
# Setup logging object with model info
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"}
litellm_logging_obj.completion_start_time = None
litellm_logging_obj.async_success_handler = AsyncMock()
request_body = {"model": "claude-sonnet-4@20250514"}
start_time = datetime.now()
passthrough_success_handler_obj = MagicMock(spec=PassThroughEndpointLogging)
url_route = "v1/projects/test-project/locations/us-east5/publishers/anthropic/models/claude-sonnet-4@20250514:streamRawPredict"
# Mock completion_cost to return a test cost value
with patch("litellm.completion_cost", return_value=0.00015):
received_chunks = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
start_time=start_time,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
):
received_chunks.append(chunk)
# Verify that cost was injected into the message_delta chunk
cost_injected = False
for chunk in received_chunks:
if isinstance(chunk, bytes):
chunk_str = chunk.decode("utf-8", errors="ignore")
if "message_delta" in chunk_str and "cost" in chunk_str:
# Parse the chunk to verify cost was added
for line in chunk_str.split("\n"):
if line.startswith("data:") and "message_delta" in line:
json_part = line.split("data:", 1)[1].strip()
if json_part and json_part != "[DONE]":
try:
obj = json.loads(json_part)
if obj.get("type") == "message_delta" and "usage" in obj and "cost" in obj["usage"]:
assert obj["usage"]["cost"] == 0.00015
cost_injected = True
except json.JSONDecodeError:
pass
assert cost_injected, "Cost was not injected into message_delta chunk"
finally:
# Restore original value
litellm.include_cost_in_streaming_usage = original_value
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_vertex_ai_anthropic_streaming_cost_injection_disabled():
"""
Test that cost is NOT injected when include_cost_in_streaming_usage is disabled.
"""
# Disable cost injection
original_value = getattr(litellm, "include_cost_in_streaming_usage", False)
litellm.include_cost_in_streaming_usage = False
try:
# Mock response with Anthropic SSE format chunks
response = AsyncMock(spec=httpx.Response)
response.status_code = 200
chunks_with_usage = [
b'data: {"type": "message_delta", "usage": {"input_tokens": 10, "output_tokens": 5}}\n\n',
]
async def mock_aiter_bytes():
for chunk in chunks_with_usage:
yield chunk
response.aiter_bytes = mock_aiter_bytes
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"}
litellm_logging_obj.completion_start_time = None
litellm_logging_obj.async_success_handler = AsyncMock()
request_body = {"model": "claude-sonnet-4@20250514"}
start_time = datetime.now()
passthrough_success_handler_obj = MagicMock(spec=PassThroughEndpointLogging)
url_route = "v1/projects/test-project/locations/us-east5/publishers/anthropic/models/claude-sonnet-4@20250514:streamRawPredict"
received_chunks = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
start_time=start_time,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
):
received_chunks.append(chunk)
# Verify that cost was NOT injected
cost_found = False
for chunk in received_chunks:
if isinstance(chunk, bytes):
chunk_str = chunk.decode("utf-8", errors="ignore")
if "cost" in chunk_str:
cost_found = True
assert not cost_found, "Cost should not be injected when feature is disabled"
finally:
# Restore original value
litellm.include_cost_in_streaming_usage = original_value
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_vertex_ai_anthropic_streaming_cost_injection_no_usage_chunk():
"""
Test that chunks without usage are not modified.
"""
original_value = getattr(litellm, "include_cost_in_streaming_usage", False)
litellm.include_cost_in_streaming_usage = True
try:
response = AsyncMock(spec=httpx.Response)
response.status_code = 200
# Chunks without usage (should not be modified)
chunks_without_usage = [
b'data: {"type": "content_block_delta", "delta": {"text": "Hello"}}\n\n',
b'data: {"type": "content_block_start", "index": 0}\n\n',
]
async def mock_aiter_bytes():
for chunk in chunks_without_usage:
yield chunk
response.aiter_bytes = mock_aiter_bytes
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"}
litellm_logging_obj.completion_start_time = None
litellm_logging_obj.async_success_handler = AsyncMock()
request_body = {"model": "claude-sonnet-4@20250514"}
start_time = datetime.now()
passthrough_success_handler_obj = MagicMock(spec=PassThroughEndpointLogging)
url_route = "v1/projects/test-project/locations/us-east5/publishers/anthropic/models/claude-sonnet-4@20250514:streamRawPredict"
received_chunks = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
start_time=start_time,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
):
received_chunks.append(chunk)
# Verify chunks remain unchanged (no cost injection attempted)
assert len(received_chunks) == len(chunks_without_usage)
# Chunks should be exactly as input since they don't contain usage
for i, chunk in enumerate(received_chunks):
assert chunk == chunks_without_usage[i]
finally:
litellm.include_cost_in_streaming_usage = original_value
@pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate")
@pytest.mark.asyncio
async def test_vertex_ai_anthropic_streaming_model_extraction():
"""
Test that model name is correctly extracted for cost calculation.
"""
original_value = getattr(litellm, "include_cost_in_streaming_usage", False)
litellm.include_cost_in_streaming_usage = True
try:
response = AsyncMock(spec=httpx.Response)
response.status_code = 200
chunks = [
b'data: {"type": "message_delta", "usage": {"input_tokens": 10, "output_tokens": 5}}\n\n',
]
async def mock_aiter_bytes():
for chunk in chunks:
yield chunk
response.aiter_bytes = mock_aiter_bytes
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {}
litellm_logging_obj.completion_start_time = None
litellm_logging_obj.async_success_handler = AsyncMock()
# Test model extraction from request body
request_body = {"model": "claude-sonnet-4@20250514"}
start_time = datetime.now()
passthrough_success_handler_obj = MagicMock(spec=PassThroughEndpointLogging)
url_route = "v1/projects/test-project/locations/us-east5/publishers/anthropic/models/claude-sonnet-4@20250514:streamRawPredict"
with patch("litellm.completion_cost") as mock_cost:
mock_cost.return_value = 0.0001
received_chunks = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
start_time=start_time,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
):
received_chunks.append(chunk)
# Verify completion_cost was called with the correct model
assert mock_cost.called
call_args = mock_cost.call_args
assert call_args[1]["model"] == "claude-sonnet-4@20250514"
finally:
litellm.include_cost_in_streaming_usage = original_value
_LIVE_ROUTE: Final = "/vertex_ai/live"
_LIVE_MODEL: Final = "gemini-live-2.5-flash"
_START: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
_END: Final = datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc)
_FIRST_TURN_USAGE: Final = {"promptTokenCount": 10, "candidatesTokenCount": 4, "totalTokenCount": 14}
_SECOND_TURN_USAGE: Final = {"promptTokenCount": 30, "candidatesTokenCount": 6, "totalTokenCount": 36}
def _logging_obj() -> LiteLLMLoggingObj:
return LiteLLMLoggingObj(
model="unknown",
messages=[{"role": "user", "content": "WebSocket connection"}],
stream=True,
call_type="pass_through_endpoint",
start_time=_START,
litellm_call_id="call-live",
function_id="websocket_passthrough",
)
def _normalize_live_session(response_body: dict | list[dict[str, object]] | None) -> dict:
return PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
httpx_response=httpx.Response(200, request=httpx.Request("GET", f"https://proxy.example.test{_LIVE_ROUTE}")),
response_body=response_body,
request_body={},
logging_obj=_logging_obj(),
url_route=_LIVE_ROUTE,
result="websocket_connection_successful",
start_time=_START,
end_time=_END,
cache_hit=False,
model=_LIVE_MODEL,
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_vertex_ai_live_route_sums_usage_of_every_turn_in_the_websocket_frames() -> None:
normalized = _normalize_live_session(
[
{"setupComplete": {}},
{"serverContent": {"modelTurn": {"parts": [{"text": "hello"}]}}},
{"usageMetadata": _FIRST_TURN_USAGE},
{"serverContent": {"modelTurn": {"parts": [{"text": "goodbye"}]}}},
{"usageMetadata": _SECOND_TURN_USAGE},
]
)
response = normalized["standard_logging_response_object"]
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (40, 10, 50)
assert response.model == _LIVE_MODEL
assert normalized["kwargs"] == {"model": _LIVE_MODEL, "custom_llm_provider": "vertex_ai"}
@pytest.mark.parametrize(
"response_body",
[
pytest.param({"usageMetadata": _FIRST_TURN_USAGE}, id="single-json-object"),
pytest.param(None, id="no-body"),
pytest.param([{"setupComplete": {}}], id="frames-without-usage"),
],
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response(
response_body: dict | list[dict[str, object]] | None,
) -> None:
normalized = _normalize_live_session(response_body)
assert normalized["standard_logging_response_object"] is None
assert normalized["kwargs"] == {"model": _LIVE_MODEL}