Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

2.5 KB
Raw Blame
import json
from collections.abc import AsyncIterable, AsyncIterator
from typing import Final

import openai
from integration._support.anthropic_sse import dropping_reply
from integration._support.wire import Reply, Request, wire_server

import litellm
from litellm.exceptions import MidStreamFallbackError

_PARTIAL_TEXT: Final = "partial answer"


def _chunk(content: str) -> bytes:
    body: Final = {
        "id": "chatcmpl-sdk-cut",
        "object": "chat.completion.chunk",
        "created": 1,
        "model": "gpt-4o-mini",
        "choices": [{"index": 0, "delta": {"role": "assistant", "content": content}, "finish_reason": None}],
    }
    return b"data: " + json.dumps(body).encode() + b"\n\n"


def _cut_after_text(request: Request) -> Reply:
    assert request.method == "POST" and "/chat/completions" in request.target, request.target
    return dropping_reply((_chunk(_PARTIAL_TEXT), _chunk(" never sent")), abort_after=1)


def _text(item: object) -> str:
    if not isinstance(item, litellm.ModelResponseStream):
        return ""
    return "".join(choice.delta.content or "" for choice in item.choices)


async def _items_or_error(stream: AsyncIterable[object]) -> AsyncIterator[object]:
    try:
        async for item in stream:
            yield item
    except openai.APIError as error:
        yield error


async def test_sdk_mid_stream_failure_debug_info_names_the_api_base_it_called() -> None:
    with wire_server(_cut_after_text) as wire:
        response: Final = await litellm.acompletion(
            model="azure/gpt-4o-mini",
            api_base=wire.url,
            api_key="synthetic-azure-key",
            api_version="2025-04-01-preview",
            messages=[{"role": "user", "content": "stream an answer"}],
            stream=True,
            num_retries=0,
        )
        assert isinstance(response, AsyncIterable), response
        items: Final = [item async for item in _items_or_error(response)]
    errors: Final = [item for item in items if isinstance(item, openai.APIError)]
    assert len(errors) == 1 and items[-1] is errors[0], items
    assert "".join(_text(item) for item in items) == _PARTIAL_TEXT, items
    error: Final = errors[0]
    mapped: Final = (
        error.original_exception
        if isinstance(error, MidStreamFallbackError) and error.original_exception is not None
        else error
    )
    debug_info: Final = getattr(mapped, "litellm_debug_info", None)
    assert isinstance(debug_info, str), mapped
    assert f"\nAPI Base: `{wire.url}`" in debug_info, debug_info