Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

6.4 KB
Raw Blame
"""
Regression test for LIT-4210: completing an A2A stream must not run the sync
success_handler on the thread-pool executor concurrently with
async_success_handler (cross-thread pydantic mutation segfaults pydantic-core).
"""

import asyncio
import time
from datetime import datetime
from typing import Final
from types import SimpleNamespace

import pytest

import litellm
from litellm.a2a_protocol import streaming_iterator as a2a_streaming_iterator_module
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging


class RecordingCustomLogger(CustomLogger):
    def __init__(self):
        super().__init__()
        self.async_hook_fired = False

    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
        self.async_hook_fired = True

    async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
        self.async_hook_fired = True


class RecordingExecutor:
    def __init__(self, inner):
        self._inner = inner
        self.submits: list = []

    def submit(self, fn, *args, **kwargs):
        self.submits.append(fn)
        return self._inner.submit(fn, *args, **kwargs)

    def submitted_for(self, logging_obj) -> list:
        return [fn for fn in self.submits if getattr(fn, "__self__", None) is logging_obj]


@pytest.mark.asyncio
async def test_custom_logger_only_never_submits_sync_success_handler(monkeypatch):
    recording_executor = RecordingExecutor(thread_pool_executor_module.executor)
    monkeypatch.setattr(thread_pool_executor_module, "executor", recording_executor)
    monkeypatch.setattr(a2a_streaming_iterator_module, "executor", recording_executor, raising=False)

    recorder = RecordingCustomLogger()
    monkeypatch.setattr(litellm, "success_callback", [recorder])
    monkeypatch.setattr(litellm, "_async_success_callback", [recorder])

    logging_obj = LitellmLogging(
        model="a2a/test-agent",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="a2a_send_message_streaming",
        start_time=time.time(),
        litellm_call_id="lit-4210-test",
        function_id="lit-4210-test",
    )

    async def _empty_stream():
        return
        yield

    iterator = A2AStreamingIterator(
        stream=_empty_stream(),
        request=SimpleNamespace(
            params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})
        ),
        logging_obj=logging_obj,
        agent_name="test-agent",
    )

    await iterator._handle_stream_complete()
    await asyncio.sleep(0.5)

    assert recorder.async_hook_fired is True
    assert recording_executor.submitted_for(logging_obj) == []


class _AgentChunk:
    def __init__(self, text: str):
        self._text = text

    def model_dump(self, mode: str, exclude_none: bool) -> dict:
        return {"result": {"kind": "message", "role": "agent", "parts": [{"kind": "text", "text": self._text}]}}


@pytest.mark.asyncio
async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
    from tests.large_text import text
    from tests.unit.litellm_core_utils.event_loop_lag import (
        assert_loop_stayed_free,
        timed_with_loop_lags,
        warm_tokenizer,
    )

    warm_tokenizer("gpt-5.6-luna")
    monkeypatch.setattr(litellm, "success_callback", [])
    monkeypatch.setattr(litellm, "_async_success_callback", [])
    logging_obj = LitellmLogging(
        model="a2a/test-agent",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="a2a_send_message_streaming",
        start_time=time.time(),
        litellm_call_id="lit-7190-test",
        function_id="lit-7190-test",
    )

    async def _stream():
        yield _AgentChunk(text * 100)

    iterator = A2AStreamingIterator(
        stream=_stream(),
        request=SimpleNamespace(
            params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": text * 100}]})
        ),
        logging_obj=logging_obj,
        agent_name="test-agent",
    )

    async def drain() -> int:
        return len([chunk async for chunk in iterator])

    yielded, took, lags = await timed_with_loop_lags(drain)

    assert yielded == 1
    usage = logging_obj.model_call_details["usage"]
    assert usage.prompt_tokens > 100_000
    assert usage.completion_tokens > 100_000
    assert_loop_stayed_free(took, lags)


class _SuccessRecorder(CustomLogger):
    def __init__(self):
        super().__init__()
        self.logged: list[tuple[datetime, object]] = []

    async def async_log_success_event(
        self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
    ) -> None:
        self.logged.append((start_time, kwargs.get("completion_start_time")))

    async def async_log_stream_event(
        self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
    ) -> None:
        self.logged.append((start_time, kwargs.get("completion_start_time")))


@pytest.mark.asyncio
async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start():
    call_start: Final = datetime(2026, 10, 9, 12, 0, 0)
    recorder: Final = _SuccessRecorder()
    logging_obj: Final = LitellmLogging(
        model="a2a/test-agent",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="a2a_send_message_streaming",
        start_time=call_start,
        litellm_call_id="a2a-first-chunk",
        function_id="a2a-first-chunk",
        dynamic_async_success_callbacks=[recorder],
    )

    async def _stream():
        yield _AgentChunk("Hel")
        yield _AgentChunk("Hello")

    iterator: Final = A2AStreamingIterator(
        stream=_stream(),
        request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})),
        logging_obj=logging_obj,
        agent_name="test-agent",
    )
    tasks_before: Final = asyncio.all_tasks()

    _ = await iterator.__anext__()
    first_chunk_at: Final = logging_obj.completion_start_time
    _ = [chunk async for chunk in iterator]
    await asyncio.gather(*(asyncio.all_tasks() - tasks_before))

    assert first_chunk_at is not None
    assert recorder.logged == [(call_start, first_chunk_at)]