Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

8.4 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
import threading
import time
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final

import openai
import pytest
from integration._support.client import eventually
from integration._support.wire import Reply, Request, Wire, wire_server

import litellm
from litellm.types.utils import ModelResponseStream

_API_KEY: Final = "synthetic-stream-release-key"
_PROMPT: Final = "does the socket go back to the pool once the stream is done with it?"
_FIRST_WORDS: Final = "hello "
_LAST_WORD: Final = "world"
_RELEASE_SECONDS: Final = 2


def _anthropic_event(event: str, payload: Mapping[str, object]) -> bytes:
    return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode()


def _openai_delta(text: str, finish_reason: str | None) -> bytes:
    chunk: Final = {
        "id": "chatcmpl-stream-release-wire",
        "object": "chat.completion.chunk",
        "created": 1700000000,
        "model": "gpt-5.6",
        "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}, "finish_reason": finish_reason}],
    }
    return b"data: " + json.dumps(chunk).encode() + b"\n\n"


_ANTHROPIC_OPENING: Final = b"".join(
    (
        _anthropic_event(
            "message_start",
            {
                "type": "message_start",
                "message": {
                    "id": "msg_stream_release_wire",
                    "type": "message",
                    "role": "assistant",
                    "model": "claude-sonnet-5-5",
                    "content": [],
                    "stop_reason": None,
                    "usage": {"input_tokens": 12, "output_tokens": 1},
                },
            },
        ),
        _anthropic_event(
            "content_block_start",
            {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
        ),
        _anthropic_event(
            "content_block_delta",
            {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": _FIRST_WORDS}},
        ),
    )
)
_ANTHROPIC_CLOSING: Final = (
    _anthropic_event(
        "content_block_delta",
        {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": _LAST_WORD}},
    ),
    _anthropic_event("content_block_stop", {"type": "content_block_stop", "index": 0}),
    _anthropic_event(
        "message_delta",
        {
            "type": "message_delta",
            "delta": {"stop_reason": "end_turn", "stop_sequence": None},
            "usage": {"output_tokens": 3},
        },
    ),
    _anthropic_event("message_stop", {"type": "message_stop"}),
)
_OPENAI_OPENING: Final = _openai_delta(_FIRST_WORDS, None)
_OPENAI_CLOSING: Final = (_openai_delta(_LAST_WORD, None), _openai_delta("", "stop"), b"data: [DONE]\n\n")


@dataclass(frozen=True, slots=True)
class _Shape:
    name: str
    model: str
    opening: bytes
    closing: tuple[bytes, ...]


_SHAPES: Final = (
    _Shape("anthropic", "anthropic/claude-sonnet-5-5", _ANTHROPIC_OPENING, _ANTHROPIC_CLOSING),
    _Shape("openai", "openai/gpt-5.6", _OPENAI_OPENING, _OPENAI_CLOSING),
    _Shape("databricks", "databricks/databricks-llama", _OPENAI_OPENING, _OPENAI_CLOSING),
)


def _peer(shape: _Shape, gate_after_first: threading.Event | None = None):
    def respond(request: Request) -> Reply:
        return Reply(
            content_type="text/event-stream", chunks=(shape.opening, *shape.closing), gate_after_first=gate_after_first
        )

    return respond


def _request_kwargs(shape: _Shape, wire: Wire) -> Mapping[str, object]:
    client: Final = openai.OpenAI(api_key=_API_KEY, base_url=wire.url) if shape.name == "openai" else None
    return {
        "model": shape.model,
        "api_base": wire.url,
        "api_key": _API_KEY,
        "messages": ({"role": "user", "content": _PROMPT},),
        "stream": True,
        **({"client": client} if client is not None else {}),
    }


def _text_of(chunk: ModelResponseStream) -> str:
    return chunk.choices[0].delta.content or ""


def _open_sessions_settling_to(wire: Wire, at_most: int) -> int:
    return eventually(wire.open_connections, lambda open_sessions: open_sessions <= at_most, seconds=_RELEASE_SECONDS)


async def _open_sessions_settling_to_while_the_loop_runs(wire: Wire, at_most: int) -> int:
    deadline: Final = time.monotonic() + _RELEASE_SECONDS
    while wire.open_connections() > at_most and time.monotonic() < deadline:
        await asyncio.sleep(0.05)
    return wire.open_connections()


@pytest.mark.parametrize("shape", _SHAPES, ids=lambda shape: shape.name)
def test_sync_stream_read_to_its_last_chunk_releases_the_connection(shape: _Shape) -> None:
    with wire_server(_peer(shape), keep_alive=True) as wire:
        request: Final = _request_kwargs(shape, wire)
        first_text: Final = "".join(_text_of(chunk) for chunk in litellm.completion(**request))
        second_text: Final = "".join(_text_of(chunk) for chunk in litellm.completion(**request))
        assert (first_text, second_text) == (_FIRST_WORDS + _LAST_WORD, _FIRST_WORDS + _LAST_WORD)
        assert _open_sessions_settling_to(wire, at_most=1) <= 1, "a finished stream left its TCP session open"


@pytest.mark.parametrize("shape", _SHAPES, ids=lambda shape: shape.name)
def test_sync_stream_closed_after_its_first_chunk_releases_the_connection(shape: _Shape) -> None:
    gate: Final = threading.Event()
    with wire_server(_peer(shape, gate_after_first=gate), keep_alive=True) as wire:
        request: Final = _request_kwargs(shape, wire)
        abandoned: Final = litellm.completion(**request)
        first_chunk: Final = next(iter(abandoned))
        assert _text_of(first_chunk) == _FIRST_WORDS
        abandoned.close()
        gate.set()
        assert _open_sessions_settling_to(wire, at_most=0) == 0, "close() left the TCP session open"
        gate.clear()
        follow_up: Final = litellm.completion(**request)
        assert _text_of(next(iter(follow_up))) == _FIRST_WORDS
        gate.set()
        assert "".join(_text_of(chunk) for chunk in follow_up) == _LAST_WORD


@pytest.mark.asyncio
async def test_async_stream_read_to_its_last_chunk_releases_the_connection() -> None:
    shape: Final = _SHAPES[0]
    with wire_server(_peer(shape), keep_alive=True) as wire:
        request: Final = _request_kwargs(shape, wire)
        first_text: Final = "".join([_text_of(chunk) async for chunk in await litellm.acompletion(**request)])
        second_text: Final = "".join([_text_of(chunk) async for chunk in await litellm.acompletion(**request)])
        assert (first_text, second_text) == (_FIRST_WORDS + _LAST_WORD, _FIRST_WORDS + _LAST_WORD)
        assert await _open_sessions_settling_to_while_the_loop_runs(wire, at_most=1) <= 1, (
            "a finished stream left its TCP session open"
        )


@pytest.mark.asyncio
async def test_async_stream_aclosed_after_its_first_chunk_releases_the_connection() -> None:
    shape: Final = _SHAPES[0]
    gate: Final = threading.Event()
    with wire_server(_peer(shape, gate_after_first=gate), keep_alive=True) as wire:
        request: Final = _request_kwargs(shape, wire)
        abandoned: Final = await litellm.acompletion(**request)
        assert _text_of(await abandoned.__anext__()) == _FIRST_WORDS
        await abandoned.aclose()
        gate.set()
        assert await _open_sessions_settling_to_while_the_loop_runs(wire, at_most=0) == 0, (
            "aclose() left the TCP session open"
        )
        gate.clear()
        follow_up: Final = await litellm.acompletion(**request)
        assert _text_of(await follow_up.__anext__()) == _FIRST_WORDS
        gate.set()
        assert "".join([_text_of(chunk) async for chunk in follow_up]) == _LAST_WORD


_ANTHROPIC_ERROR_MIDDLE: Final = _anthropic_event(
    "error", {"type": "error", "error": {"type": "overloaded_error", "message": "peer overloaded mid stream"}}
)


def test_sync_stream_that_raises_mid_stream_releases_the_connection() -> None:
    shape: Final = _SHAPES[0]
    broken: Final = _Shape(shape.name, shape.model, shape.opening, (_ANTHROPIC_ERROR_MIDDLE, *shape.closing))
    with wire_server(_peer(broken), keep_alive=True) as wire:
        request: Final = _request_kwargs(broken, wire)
        with pytest.raises(Exception, match="overloaded"):
            "".join(_text_of(chunk) for chunk in litellm.completion(**request))
        assert _open_sessions_settling_to(wire, at_most=0) == 0, (
            "a stream that raised mid-way left its TCP session open"
        )