Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

7.3 KB
Raw Blame
import json
from collections.abc import Mapping
from typing import Final

import httpx
import pytest
import respx

import litellm


def _sse_body(chunks: tuple[Mapping[str, object], ...]) -> str:
    return "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n"


@pytest.mark.asyncio
@respx.mock
async def test_async_gemini_streaming_tool_call_arguments_are_valid(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
    chunks: Final = (
        {
            "candidates": [
                {
                    "content": {
                        "parts": [
                            {
                                "functionCall": {
                                    "name": "generate_series_of_questions",
                                    "args": {
                                        "questions": [
                                            "What is a bridge?",
                                            "How is it designed?",
                                            "How is it built?",
                                        ]
                                    },
                                }
                            }
                        ],
                        "role": "model",
                    },
                    "index": 0,
                }
            ],
            "usageMetadata": {
                "promptTokenCount": 5,
                "candidatesTokenCount": 7,
                "totalTokenCount": 12,
            },
        },
        {
            "candidates": [
                {
                    "content": {"parts": [], "role": "model"},
                    "finishReason": "STOP",
                    "index": 0,
                }
            ],
            "usageMetadata": {
                "promptTokenCount": 5,
                "candidatesTokenCount": 7,
                "totalTokenCount": 12,
            },
        },
    )
    route: Final = respx.post(
        url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-lite:streamGenerateContent.*"
    ).mock(
        return_value=httpx.Response(
            200,
            text=_sse_body(chunks),
            headers={"content-type": "text/event-stream"},
        )
    )

    stream: Final = await litellm.acompletion(
        model="gemini/gemini-2.5-flash-lite",
        messages=[
            {"role": "system", "content": "You are an AI assistant"},
            {"role": "user", "content": "Generate three questions about civil engineering."},
        ],
        tools=[
            {
                "type": "function",
                "function": {
                    "name": "generate_series_of_questions",
                    "description": "Generate questions",
                    "parameters": {
                        "type": "object",
                        "properties": {
                            "questions": {
                                "type": "array",
                                "items": {"type": "string"},
                            }
                        },
                        "required": ["questions"],
                    },
                },
            }
        ],
        stream=True,
        api_key="test-gemini-key",
    )
    response_chunks: Final = [chunk async for chunk in stream]
    tool_calls: Final = [
        tool_call
        for chunk in response_chunks
        for tool_call in chunk.choices[0].delta.tool_calls or ()
    ]

    assert route.call_count == 1
    assert response_chunks[0].choices[0].delta.role == "assistant"
    assert tuple(chunk.choices[0].finish_reason for chunk in response_chunks) == (None, "tool_calls")
    assert len(tool_calls) == 1
    assert tool_calls[0].function.name == "generate_series_of_questions"
    assert json.loads(tool_calls[0].function.arguments) == {
        "questions": [
            "What is a bridge?",
            "How is it designed?",
            "How is it built?",
        ]
    }


@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
@respx.mock
async def test_gemini_streaming_legacy_function_call_arguments_are_valid(
    sync_mode: bool,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
    chunks: Final = (
        {
            "candidates": [
                {
                    "content": {
                        "parts": [
                            {
                                "functionCall": {
                                    "name": "get_current_weather",
                                    "args": {"location": "Boston", "unit": "fahrenheit"},
                                }
                            }
                        ],
                        "role": "model",
                    },
                    "index": 0,
                }
            ]
        },
        {
            "candidates": [
                {
                    "content": {"parts": [], "role": "model"},
                    "finishReason": "STOP",
                    "index": 0,
                }
            ]
        },
    )
    respx.post(
        url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-lite:streamGenerateContent.*"
    ).mock(
        return_value=httpx.Response(
            200,
            text=_sse_body(chunks),
            headers={"content-type": "text/event-stream"},
        )
    )
    messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}]
    functions: Final = [
        {
            "name": "get_current_weather",
            "description": "Get the current weather",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {"type": "string"},
                    "unit": {"type": "string"},
                },
                "required": ["location", "unit"],
            },
        }
    ]
    if sync_mode:
        stream_chunks: Final = list(
            litellm.completion(
                model="gemini/gemini-2.5-flash-lite",
                messages=messages,
                functions=functions,
                stream=True,
                api_key="test-gemini-key",
            )
        )
    else:
        stream: Final = await litellm.acompletion(
            model="gemini/gemini-2.5-flash-lite",
            messages=messages,
            functions=functions,
            stream=True,
            api_key="test-gemini-key",
        )
        stream_chunks: Final = [chunk async for chunk in stream]

    function_calls: Final = tuple(
        chunk.choices[0].delta.function_call
        for chunk in stream_chunks
        if chunk.choices and chunk.choices[0].delta.function_call is not None
    )
    assert tuple(
        function_call.name for function_call in function_calls if function_call.name is not None
    ) == ("get_current_weather",)
    arguments: Final = "".join(function_call.arguments or "" for function_call in function_calls)
    assert json.loads(arguments) == {"location": "Boston", "unit": "fahrenheit"}
    rebuilt: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages)
    assert rebuilt.choices[0].message.function_call.name == "get_current_weather"
    assert json.loads(rebuilt.choices[0].message.function_call.arguments) == {"location": "Boston", "unit": "fahrenheit"}