The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import re
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import Final
from urllib.parse import unquote
import pytest
from integration._support.upstream import _aws_event_frame
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
import litellm
from litellm.exceptions import BadGatewayError, BadRequestError, MidStreamFallbackError
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
_MODEL_ID: Final = "global.moonshotai.kimi-k3"
_CONVERSE_MODEL: Final = f"bedrock/converse/{_MODEL_ID}"
_STREAM_TARGET: Final = f"/model/{_MODEL_ID}/converse-stream"
_EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
_ANSWER: Final = "bedrock sync decoder control"
_REJECTION: Final = "structured output schema uses unsupported regex negative look-ahead"
_UNKNOWN_TYPE: Final = "somethingBedrockAddedLater"
_USAGE: Final[dict[str, JsonValue]] = {"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}}
def _frame(event_type: str, payload: Mapping[str, JsonValue]) -> bytes:
return _aws_event_frame(event_type, payload, "sc", "u")
_NORMAL: Final = b"".join(
(
_frame("messageStart", {"role": "assistant"}),
_frame("contentBlockDelta", {"delta": {"text": _ANSWER}, "contentBlockIndex": 0}),
_frame("contentBlockStop", {"contentBlockIndex": 0}),
_frame("messageStop", {"stopReason": "end_turn"}),
_frame("metadata", _USAGE),
)
)
_VALIDATION_FRAME: Final = _frame("validationException", {"message": _REJECTION})
_UNKNOWN_FRAME: Final = _frame(_UNKNOWN_TYPE, {"future": True})
@dataclass(frozen=True, slots=True)
class _Consumed:
text: str
finish_reasons: tuple[str | None, ...]
def _peer(frames: bytes) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert unquote(request.target) == _STREAM_TARGET, request.target
return Reply(body=frames, content_type=_EVENT_STREAM)
return respond
def _consume_sync_stream(wire: Wire) -> _Consumed:
response: Final = litellm.completion(
model=_CONVERSE_MODEL,
messages=[{"role": "user", "content": "What does the sync decoder do with this stream?"}],
max_tokens=16,
stream=True,
api_base=wire.url,
aws_access_key_id="AKIASCRIPTEDPROVIDER",
aws_secret_access_key="scripted-secret",
aws_region_name="us-east-1",
num_retries=0,
)
assert isinstance(response, CustomStreamWrapper), type(response)
chunks: Final = tuple(response)
return _Consumed(
text="".join(str(chunk.choices[0].delta.content or "") for chunk in chunks),
finish_reasons=tuple(chunk.choices[0].finish_reason for chunk in chunks),
)
def test_k01_sync_stream_with_a_validation_exception_frame_first_raises_a_400() -> None:
with wire_server(_peer(_VALIDATION_FRAME)) as wire:
with pytest.raises(BadRequestError, match=re.escape(_REJECTION)) as raised:
_consume_sync_stream(wire)
assert raised.value.status_code == 400, raised.value
assert len(wire.drain()) == 1
def test_k02_sync_stream_whose_only_frame_has_an_unknown_event_type_raises_a_502_naming_it() -> None:
with wire_server(_peer(_UNKNOWN_FRAME)) as wire:
with pytest.raises(MidStreamFallbackError, match=re.escape(_UNKNOWN_TYPE)) as raised:
_consume_sync_stream(wire)
assert raised.value.status_code == 502, raised.value
assert isinstance(raised.value.original_exception, BadGatewayError), raised.value.original_exception
assert "none of its 1 events carried a known event type" in str(raised.value), raised.value
assert len(wire.drain()) == 1
def test_k03_sync_stream_with_normal_frames_delivers_the_text_and_a_stop() -> None:
with wire_server(_peer(_NORMAL)) as wire:
consumed: Final = _consume_sync_stream(wire)
assert consumed.text == _ANSWER, consumed
assert consumed.finish_reasons[-1] == "stop", consumed
assert len(wire.drain()) == 1
def test_k04_sync_stream_open_400_carries_the_bedrock_body_as_text() -> None:
body: Final = '{"message":"The request is invalid."}'
def reject(_: Request) -> Reply:
return Reply(status=400, body=body.encode())
with wire_server(reject) as wire:
with pytest.raises(BadRequestError) as raised:
_consume_sync_stream(wire)
assert raised.value.status_code == 400, raised.value
assert f"BedrockException - {body}" in raised.value.message, raised.value.message
assert len(wire.drain()) == 1