Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

61.8 KB
Raw Blame
"""Regression tests for PassThroughStreamingHandler.chunk_processor."""

import asyncio
import json
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from datetime import datetime
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import pytest
import respx

import litellm
import litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler as tinyfish_handler_module
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import (
    mark_sse_poller_spawned,
    sse_poller_spawned,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt
from litellm.proxy.pass_through_endpoints.streaming_handler import (
    PassThroughStreamingHandler,
)
from litellm.proxy.utils import _failure_fields_to_lift
from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType


def _make_streaming_response(chunks):
    mock = MagicMock(spec=httpx.Response)
    mock.status_code = 200

    async def _aiter_bytes():
        for c in chunks:
            yield c

    mock.aiter_bytes = _aiter_bytes
    return mock


def _unarmed_logging_obj():
    """Real Logging objects only carry _on_deferred_stream_complete when the
    proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
    and would spuriously trigger the deferral branch."""
    obj = MagicMock()
    obj._on_deferred_stream_complete = None
    return obj


@pytest.mark.asyncio
async def test_chunk_processor_logs_on_normal_completion():
    chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
    response = _make_streaming_response(chunks)

    mock_logging_obj = _unarmed_logging_obj()
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ) as mock_route:
        received = []
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        ):
            received.append(chunk)

        await asyncio.sleep(0)

    assert received == chunks
    mock_route.assert_called_once()
    call_kwargs = mock_route.call_args.kwargs
    assert call_kwargs["raw_bytes"] == chunks


@pytest.mark.asyncio
async def test_chunk_processor_logs_on_client_disconnect():
    chunks = [b"event-1", b"event-2", b"event-3"]
    response = _make_streaming_response(chunks)

    mock_logging_obj = _unarmed_logging_obj()
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ) as mock_route:
        gen = PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        )

        first = await gen.__anext__()
        await gen.aclose()

        await asyncio.sleep(0)

    assert first == chunks[0]
    mock_route.assert_called_once()
    call_kwargs = mock_route.call_args.kwargs
    assert call_kwargs["raw_bytes"] == [chunks[0]]


@pytest.mark.asyncio
async def test_chunk_processor_does_not_schedule_success_logging_for_upstream_error():
    """A 4xx/5xx upstream response is already logged as a failure by the caller
    before this generator starts; scheduling success logging here too would
    double-log the same request in SpendLogs."""
    chunks = [b'{"error": "denied"}']
    response = _make_streaming_response(chunks)
    response.status_code = 403

    mock_logging_obj = _unarmed_logging_obj()
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ) as mock_route:
        received = []
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        ):
            received.append(chunk)

        await asyncio.sleep(0)

    assert received == chunks
    mock_route.assert_not_called()


@pytest.mark.asyncio
async def test_chunk_processor_does_not_schedule_logging_when_no_chunks():
    response = _make_streaming_response([])

    mock_logging_obj = _unarmed_logging_obj()
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ) as mock_route:
        received = []
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        ):
            received.append(chunk)

    assert received == []
    mock_route.assert_not_called()


@pytest.mark.asyncio
async def test_chunk_processor_routes_logging_through_logging_worker():
    """The spend-log coroutine must be handed to the durable logging worker, which
    keeps a strong reference and drains on shutdown, instead of a bare
    asyncio.create_task that the event loop only weak-references and can drop
    under GC/load, silently losing the SpendLogs row for a successful call."""
    chunks = [b"chunk-1", b"chunk-2"]
    response = _make_streaming_response(chunks)

    enqueued = []

    def _capture(async_coroutine):
        enqueued.append(async_coroutine)
        async_coroutine.close()

    with (
        patch.object(
            PassThroughStreamingHandler,
            "route_streaming_logging_to_handler",
            new=AsyncMock(),
        ),
        patch.object(
            GLOBAL_LOGGING_WORKER,
            "ensure_initialized_and_enqueue",
            side_effect=_capture,
        ) as mock_enqueue,
    ):
        received = []
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=_unarmed_logging_obj(),
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        ):
            received.append(chunk)

    assert received == chunks
    mock_enqueue.assert_called_once()
    assert asyncio.iscoroutine(enqueued[0])


@pytest.mark.asyncio
async def test_chunk_processor_routes_logging_through_logging_worker_on_disconnect():
    """Even when the client disconnects mid-stream, the partial-usage log must go
    through the durable logging worker rather than a droppable bare task."""
    chunks = [b"event-1", b"event-2", b"event-3"]
    response = _make_streaming_response(chunks)

    enqueued = []

    def _capture(async_coroutine):
        enqueued.append(async_coroutine)
        async_coroutine.close()

    with (
        patch.object(
            PassThroughStreamingHandler,
            "route_streaming_logging_to_handler",
            new=AsyncMock(),
        ),
        patch.object(
            GLOBAL_LOGGING_WORKER,
            "ensure_initialized_and_enqueue",
            side_effect=_capture,
        ) as mock_enqueue,
    ):
        gen = PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=_unarmed_logging_obj(),
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/bedrock/model/claude/invoke-with-response-stream",
        )
        await gen.__anext__()
        await gen.aclose()

    mock_enqueue.assert_called_once()
    assert asyncio.iscoroutine(enqueued[0])


def _logging_obj_with_write_once_cst():
    """Build a MagicMock that mirrors the real Logging behavior: update_completion_start_time
    latches self.completion_start_time so the write-once guard actually latches."""
    obj = _unarmed_logging_obj()
    obj.completion_start_time = None

    def _update(*, completion_start_time):
        obj.completion_start_time = completion_start_time

    obj.update_completion_start_time.side_effect = _update
    return obj


@pytest.mark.asyncio
async def test_chunk_processor_stamps_completion_start_time_on_first_chunk():
    """Regression: LIT-4185 — streaming pass-through must stamp completion_start_time on
    the first upstream chunk. Otherwise _success_handler_helper_fn falls back to
    completion_start_time = end_time, and Prometheus/OTEL/SpendLogs TTFT reads as
    total request duration (0 speedup)."""
    chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
    response = _make_streaming_response(chunks)

    mock_logging_obj = _logging_obj_with_write_once_cst()
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ):
        received = []
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-haiku-4-5"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/v1/messages",
        ):
            received.append(chunk)

        await asyncio.sleep(0)

    assert received == chunks
    mock_logging_obj.update_completion_start_time.assert_called_once()
    stamped = mock_logging_obj.update_completion_start_time.call_args.kwargs["completion_start_time"]
    assert isinstance(stamped, datetime)


@pytest.mark.asyncio
async def test_chunk_processor_does_not_reset_completion_start_time_on_later_chunks():
    """The stamp must be write-once: reading it on chunk 2/3 must not overwrite a real TTFT
    from chunk 1 (which would collapse TTFT to time-to-last-chunk)."""
    chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
    response = _make_streaming_response(chunks)

    real_first = datetime(2020, 1, 1, 0, 0, 0)
    mock_logging_obj = _unarmed_logging_obj()
    # Simulate first-chunk stamp having already landed (e.g. under contention or a
    # prior wrapper that already set it): later chunks must be no-ops.
    mock_logging_obj.completion_start_time = real_first
    mock_passthrough_handler = MagicMock()

    with patch.object(
        PassThroughStreamingHandler,
        "route_streaming_logging_to_handler",
        new=AsyncMock(),
    ):
        async for _ in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-haiku-4-5"},
            litellm_logging_obj=mock_logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=mock_passthrough_handler,
            url_route="/v1/messages",
        ):
            pass

    mock_logging_obj.update_completion_start_time.assert_not_called()
    assert mock_logging_obj.completion_start_time == real_first


@pytest.mark.asyncio
async def test_chunk_processor_stamps_completion_start_time_on_cost_injection_path():
    """The cost-injection branch runs alongside a hot path; both must stamp TTFT."""
    import litellm as litellm_mod

    chunks = [b"event: message_start\ndata: {}\n\n"]
    response = _make_streaming_response(chunks)

    mock_logging_obj = _logging_obj_with_write_once_cst()
    mock_logging_obj.model_call_details = {"model": "claude-haiku-4-5"}
    mock_passthrough_handler = MagicMock()

    original = getattr(litellm_mod, "include_cost_in_streaming_usage", False)
    litellm_mod.include_cost_in_streaming_usage = True
    try:
        with patch.object(
            PassThroughStreamingHandler,
            "route_streaming_logging_to_handler",
            new=AsyncMock(),
        ):
            async for _ in PassThroughStreamingHandler.chunk_processor(
                response=response,
                request_body={"model": "claude-haiku-4-5"},
                litellm_logging_obj=mock_logging_obj,
                endpoint_type=EndpointType.ANTHROPIC,
                start_time=datetime.now(),
                passthrough_success_handler_obj=mock_passthrough_handler,
                url_route="/v1/messages",
            ):
                pass
    finally:
        litellm_mod.include_cost_in_streaming_usage = original

    mock_logging_obj.update_completion_start_time.assert_called_once()


def _openai_passthrough_stream_chunks():
    return [
        (
            b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk",'
            b'"choices":[{"index":0,"delta":{"content":"Hi"}}],"usage":null}\n\n'
        ),
        b": keepalive\n\n",
        (
            b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[],'
            b'"usage":{"prompt_tokens":11,"completion_tokens":4,"total_tokens":15,'
            b'"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},'
            b'"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,'
            b'"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}\n\n'
        ),
        b"data: [DONE]\n\n",
    ]


async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
    response = _make_streaming_response(chunks)
    received = []
    async for chunk in PassThroughStreamingHandler.chunk_processor(
        response=response,
        request_body={"model": "gpt-4o-mini", "stream": True},
        litellm_logging_obj=_unarmed_logging_obj(),
        endpoint_type=endpoint_type,
        start_time=datetime(2026, 1, 1),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/openai/v1/chat/completions",
        route_streaming_logging=AsyncMock(),
    ):
        received.append(chunk)
    await asyncio.sleep(0)
    return received


@pytest.mark.asyncio
async def test_chunk_processor_injects_cost_into_openai_passthrough_usage_frame(monkeypatch):
    """Regression: issue #36492 — with include_cost_in_streaming_usage on, the final
    OpenAI passthrough chat.completion.chunk usage frame must carry usage.cost, like
    every other streaming surface already does."""
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
    chunks = _openai_passthrough_stream_chunks()

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)

    assert received[0] == chunks[0]
    assert received[1] == chunks[1]
    assert received[3] == chunks[3]
    final_payload = json.loads(received[2].decode("utf-8").split("data:", 1)[1].strip())
    pricing = litellm.model_cost["gpt-4o-mini"]
    expected_cost = 11 * pricing["input_cost_per_token"] + 4 * pricing["output_cost_per_token"]
    assert final_payload["usage"]["cost"] == pytest.approx(expected_cost)
    assert final_payload["usage"]["cost"] > 0
    assert final_payload["usage"]["prompt_tokens"] == 11
    assert final_payload["usage"]["completion_tokens"] == 4
    assert final_payload["usage"]["total_tokens"] == 15


@pytest.mark.asyncio
async def test_chunk_processor_injects_cost_into_usage_frame_fragmented_across_chunks(monkeypatch):
    """Regression: an SSE usage frame split across transport chunks must still get
    cost injected once the frame completes, instead of passing through untouched."""
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
    whole = _openai_passthrough_stream_chunks()
    usage_frame = whole[2]
    split_at = len(usage_frame) // 2
    chunks = [whole[0], whole[1], usage_frame[:split_at], usage_frame[split_at:], whole[3]]

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)

    reassembled = b"".join(received).decode("utf-8")
    usage_lines = [ln for ln in reassembled.split("\n") if '"total_tokens"' in ln]
    assert len(usage_lines) == 1
    final_payload = json.loads(usage_lines[0].split("data:", 1)[1].strip())
    assert final_payload["usage"]["cost"] > 0
    assert final_payload["usage"]["prompt_tokens"] == 11
    assert reassembled.endswith("data: [DONE]\n\n")


@pytest.mark.asyncio
async def test_chunk_processor_streams_crlf_delimited_frames_live_and_injects_cost(monkeypatch):
    """Regression: CRLF-delimited SSE frames must flow as they complete instead of
    buffering until EOF, and the usage frame must still get cost injected."""
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
    chunks = [chunk.replace(b"\n\n", b"\r\n\r\n") for chunk in _openai_passthrough_stream_chunks()]

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)

    assert len(received) == len(chunks)
    assert received[0] == chunks[0]
    injected_usage_frame = received[2]
    assert injected_usage_frame.endswith(b"\r\n\r\n")
    assert b"\n" not in injected_usage_frame.replace(b"\r\n", b"")
    reassembled = b"".join(received).decode("utf-8")
    usage_lines = [ln for ln in reassembled.replace("\r\n", "\n").split("\n") if '"total_tokens"' in ln]
    assert len(usage_lines) == 1
    final_payload = json.loads(usage_lines[0].split("data:", 1)[1].strip())
    assert final_payload["usage"]["cost"] > 0


@pytest.mark.asyncio
async def test_chunk_processor_flag_off_leaves_openai_passthrough_stream_byte_identical(monkeypatch):
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
    chunks = _openai_passthrough_stream_chunks()

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)

    assert received == chunks


@pytest.mark.asyncio
async def test_chunk_processor_flag_on_leaves_openai_frames_without_usage_untouched(monkeypatch):
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
    chunks = [
        (
            b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk",'
            b'"choices":[{"index":0,"delta":{"content":"Hi"}}],"usage":null}\n\n'
        ),
        b": keepalive\n\n",
        b"not json at all\n\n",
        b"data: [DONE]\n\n",
    ]

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)

    assert received == chunks


@pytest.mark.asyncio
async def test_chunk_processor_flag_on_leaves_generic_passthrough_untouched(monkeypatch):
    monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
    chunks = _openai_passthrough_stream_chunks()

    received = await _collect_openai_passthrough_chunks(chunks, EndpointType.GENERIC)

    assert received == chunks


def test_convert_raw_bytes_survives_truncated_multibyte_sequence():
    """A stream cut mid-multibyte-sequence (client disconnect) must still decode
    via errors="replace" so the usage events already received are logged, instead
    of raising UnicodeDecodeError and dropping the whole request from SpendLogs."""
    # the 3-byte "☃" (E2 98 83) is cut after 2 bytes, leaving an invalid sequence
    # that strict utf-8 decode would raise on, discarding the message_delta line too
    truncated_codepoint = "☃".encode("utf-8")[:2]
    raw_bytes = [
        b'data: {"text": "' + truncated_codepoint,
        b'\ndata: {"type": "message_delta"}\n',
    ]

    lines = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)

    assert any('"type": "message_delta"' in line for line in lines)


@pytest.mark.asyncio
async def test_chunk_processor_defers_logging_until_fire_when_armed():
    """Regression for PR #38722: native /v1/messages streams route through
    chunk_processor, which enqueued the spend log the moment the stream ended,
    racing the guardrail end-of-stream scan and logging
    guardrail_information as null. With deferred dispatch armed, the completed
    stream must park the logging coroutine on logging_obj and only enqueue it
    when ProxyLogging._fire_deferred_stream_logging fires after the scan."""
    from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
    from litellm.proxy.utils import ProxyLogging

    chunks = [b"event-1", b"event-2"]
    response = _make_streaming_response(chunks)

    logging_obj = _unarmed_logging_obj()
    logging_obj._deferred_stream_complete_args = None

    enqueued = []

    def _capture(async_coroutine):
        enqueued.append(async_coroutine)
        async_coroutine.close()

    with patch.object(  # test-quality-ok: GLOBAL_LOGGING_WORKER is a process-global singleton with no injection seam
        GLOBAL_LOGGING_WORKER,
        "ensure_initialized_and_enqueue",
        side_effect=_capture,
    ) as mock_enqueue:
        gen = PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/v1/messages",
            route_streaming_logging=AsyncMock(),
        )
        ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
            response=gen,
            route_type="anthropic_messages",
            user_api_key_dict=MagicMock(),
            logging_obj=logging_obj,
        )

        received = []
        async for chunk in gen:
            received.append(chunk)
        await asyncio.sleep(0)

        assert received == chunks
        mock_enqueue.assert_not_called()
        parked = logging_obj._deferred_stream_complete_args
        assert isinstance(parked, tuple) and len(parked) == 1
        assert asyncio.iscoroutine(parked[0])

        ProxyLogging._fire_deferred_stream_logging({"litellm_logging_obj": logging_obj})
        await asyncio.sleep(0)

        mock_enqueue.assert_called_once()


@pytest.mark.asyncio
async def test_chunk_processor_enqueues_immediately_on_disconnect_even_when_armed():
    """Client disconnects never reach _fire_deferred_stream_logging, so parking
    the coroutine there would lose the partial-usage spend log (LIT-2642); the
    disconnect path must keep enqueueing immediately."""
    chunks = [b"event-1", b"event-2", b"event-3"]
    response = _make_streaming_response(chunks)

    logging_obj = _unarmed_logging_obj()

    async def _armed_closure(logging_coroutine):
        raise AssertionError("deferred closure must not fire on disconnect")

    logging_obj._on_deferred_stream_complete = _armed_closure
    logging_obj._deferred_stream_complete_args = None

    enqueued = []

    def _capture(async_coroutine):
        enqueued.append(async_coroutine)
        async_coroutine.close()

    with patch.object(  # test-quality-ok: GLOBAL_LOGGING_WORKER is a process-global singleton with no injection seam
        GLOBAL_LOGGING_WORKER,
        "ensure_initialized_and_enqueue",
        side_effect=_capture,
    ) as mock_enqueue:
        gen = PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/v1/messages",
            route_streaming_logging=AsyncMock(),
        )
        await gen.__anext__()
        await gen.aclose()

    mock_enqueue.assert_called_once()
    assert logging_obj._deferred_stream_complete_args is None


class _EventRecorder(CustomLogger):
    def __init__(self):
        super().__init__()
        self.failure_kwargs = []
        self.success_kwargs = []

    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
        self.failure_kwargs.append(kwargs)

    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
        self.success_kwargs.append(kwargs)


def _anthropic_sse(event: str, payload: dict) -> bytes:
    return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode()


def _anthropic_stream_that_times_out_mid_stream():
    mock = MagicMock(spec=httpx.Response)
    mock.status_code = 200

    async def _aiter_bytes():
        yield _anthropic_sse(
            "message_start",
            {
                "type": "message_start",
                "message": {
                    "id": "msg_1",
                    "type": "message",
                    "role": "assistant",
                    "model": "claude-sonnet-5",
                    "content": [],
                    "stop_reason": None,
                    "usage": {"input_tokens": 52, "output_tokens": 1},
                },
            },
        )
        yield _anthropic_sse(
            "content_block_start",
            {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
        )
        yield _anthropic_sse(
            "content_block_delta",
            {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}},
        )
        raise httpx.ReadTimeout("Timeout on reading data from socket")

    mock.aiter_bytes = _aiter_bytes
    return mock


@pytest.mark.asyncio
async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception():
    """A stream that dies after the first chunks is a failed request: the failure
    callbacks must fire once with the partial usage and cost, and the success
    routing must never run for it."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime(2026, 1, 1),
        litellm_call_id="test-mid-stream-timeout",
        function_id="test-mid-stream-timeout",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )
    success_routes = []

    async def _record_success_route(**kwargs):
        success_routes.append(kwargs)

    received = []

    async def _consume_stream():
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=_anthropic_stream_that_times_out_mid_stream(),
            request_body={"model": "claude-sonnet-5", "stream": True},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/v1/messages",
            route_streaming_logging=_record_success_route,
        ):
            received.append(chunk)

    with pytest.raises(httpx.ReadTimeout):
        await _consume_stream()

    for _ in range(300):
        if recorder.failure_kwargs:
            break
        await asyncio.sleep(0.01)

    assert len(received) == 3
    assert success_routes == []
    assert recorder.success_kwargs == []
    assert len(recorder.failure_kwargs) == 1
    failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
    assert failure_payload["status"] == "failure"
    assert failure_payload["prompt_tokens"] == 52
    assert failure_payload["response_cost"] > 0
    assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)


def _anthropic_stream_dropping_after(frames: tuple[bytes, ...]) -> httpx.Response:
    mock: Final = MagicMock(spec=httpx.Response)
    mock.status_code = 200

    async def _aiter_bytes() -> AsyncIterator[bytes]:
        for frame in frames:
            yield frame
        raise httpx.ReadError("upstream closed the connection")

    mock.aiter_bytes = _aiter_bytes
    return mock


def _anthropic_messages_attempt(
    logging_obj: LiteLLMLoggingObj, response: httpx.Response
) -> AsyncGenerator[bytes, None]:
    return PassThroughStreamingHandler.chunk_processor(
        response=response,
        request_body={"model": "claude-sonnet-5", "stream": True},
        litellm_logging_obj=logging_obj,
        endpoint_type=EndpointType.ANTHROPIC,
        start_time=datetime(2026, 1, 1),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/v1/messages",
    )


def _anthropic_messages_logging_obj(recorder: _EventRecorder, call_id: str) -> LiteLLMLoggingObj:
    return LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime(2026, 1, 1),
        litellm_call_id=call_id,
        function_id=call_id,
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )


async def _drop_after_message_start(logging_obj: LiteLLMLoggingObj, message_id: str, input_tokens: int) -> None:
    dropped: Final = _anthropic_messages_attempt(
        logging_obj, _anthropic_stream_dropping_after((_anthropic_message_start_frame(message_id, input_tokens),))
    )
    with pytest.raises(httpx.ReadError):
        async for _ in dropped:
            pass


@pytest.mark.asyncio
async def test_dropped_attempt_logs_its_failure_without_claiming_the_shared_logging_object() -> None:
    """The router retries a /v1/messages stream the provider drops after message_start on the same logging
    object. The dropped attempt's failure log bills its own usage and leaves the shared object free: the
    retry's success log carries the retry's usage and no stale exception, while the stash stays for the
    failure row of a request that runs out of retries until the router opens the next attempt."""
    recorder: Final = _EventRecorder()
    logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-dropped-attempt")

    await _drop_after_message_start(logging_obj, "msg_a1", 52)
    await _settle(recorder.failure_kwargs)

    assert recorder.failure_kwargs[0]["standard_logging_object"]["prompt_tokens"] == 52
    assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadError)
    assert logging_obj.model_call_details.get("has_logged_async_failure") is not True
    assert "exception" not in logging_obj.model_call_details
    stashed: Final = logging_obj.model_call_details.get("combined_usage_object")
    assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 52

    logging_obj.carry_failed_attempt_usage()
    retry: Final = _anthropic_messages_attempt(
        logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
    )
    async for _ in retry:
        pass
    await _settle(recorder.success_kwargs)

    success_payload: Final = recorder.success_kwargs[0]["standard_logging_object"]
    assert success_payload["status"] == "success"
    assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
    assert recorder.success_kwargs[0].get("exception") is None
    assert len(recorder.failure_kwargs) == 1


@pytest.mark.asyncio
async def test_every_dropped_attempt_logs_its_own_failure() -> None:
    """Two attempts of one /v1/messages request dropped in a row each log a failure billing their own usage;
    the first attempt's log must not swallow the second's, and the shared object ends up holding the last
    attempt's usage for the request's failure row."""
    recorder: Final = _EventRecorder()
    logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-every-dropped-attempt")

    await _drop_after_message_start(logging_obj, "msg_a1", 52)
    await _settle(recorder.failure_kwargs)
    logging_obj.carry_failed_attempt_usage()
    await _drop_after_message_start(logging_obj, "msg_a2", 9)
    await _settle(recorder.failure_kwargs, count=2)

    assert [kwargs["standard_logging_object"]["prompt_tokens"] for kwargs in recorder.failure_kwargs] == [52, 9]
    assert all(isinstance(kwargs["exception"], httpx.ReadError) for kwargs in recorder.failure_kwargs)
    stashed: Final = logging_obj.model_call_details.get("combined_usage_object")
    assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 9

def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes:
    return _anthropic_sse(
        "message_start",
        {
            "type": "message_start",
            "message": {
                "id": message_id,
                "type": "message",
                "role": "assistant",
                "model": "claude-sonnet-5",
                "content": [],
                "stop_reason": None,
                "stop_sequence": None,
                "usage": {"input_tokens": input_tokens, "output_tokens": 1},
            },
        },
    )


def _anthropic_completed_stream_tail() -> tuple[bytes, ...]:
    return (
        _anthropic_sse(
            "content_block_start",
            {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
        ),
        _anthropic_sse(
            "content_block_delta",
            {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
        ),
        _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
        _anthropic_sse(
            "message_delta",
            {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 3}},
        ),
        _anthropic_sse("message_stop", {"type": "message_stop"}),
    )


async def _settle(recorded: Sequence[Mapping[str, object]], count: int = 1) -> None:
    for _ in range(300):
        if len(recorded) >= count:
            return
        await asyncio.sleep(0.01)


def _anthropic_overloaded_error_frame() -> bytes:
    return _anthropic_sse("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}})


def _anthropic_stream_of(frames: tuple[bytes, ...]):
    mock = MagicMock(spec=httpx.Response)
    mock.status_code = 200

    async def _aiter_bytes():
        for frame in frames:
            yield frame

    mock.aiter_bytes = _aiter_bytes
    return mock


@pytest.mark.parametrize(
    "frames, billed",
    [
        pytest.param(
            (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()), True, id="error-after-message-start"
        ),
        pytest.param((_anthropic_overloaded_error_frame(),), False, id="error-frame-only"),
    ],
)
@pytest.mark.asyncio
async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_closed_at_the_frame(frames, billed):
    """The router retries a /v1/messages stream whose provider sent `event: error` before any content by closing
    it at the frame. That attempt never completed: it logs once as a failure carrying whatever usage the provider
    had reported, and the request's one-shot success log stays free for the retry that shares the logging object."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime.now(),
        litellm_call_id="test-error-frame",
        function_id="test-error-frame",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )
    stream = PassThroughStreamingHandler.chunk_processor(
        response=_anthropic_stream_of(frames),
        request_body={"model": "claude-sonnet-5", "stream": True},
        litellm_logging_obj=logging_obj,
        endpoint_type=EndpointType.ANTHROPIC,
        start_time=datetime.now(),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/v1/messages",
    )
    received = []
    async for chunk in stream:
        received.append(chunk)
        if b"event: error" in chunk:
            break
    await aclose_superseded_attempt(stream)
    await _settle(recorder.failure_kwargs)

    assert received == list(frames)
    assert recorder.success_kwargs == []
    assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True
    assert logging_obj.should_run_logging("async_failure")
    assert len(recorder.failure_kwargs) == 1
    failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
    assert failure_payload["status"] == "failure"
    assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
    if billed:
        assert failure_payload["prompt_tokens"] == 52
        assert failure_payload["response_cost"] > 0
    else:
        assert failure_payload["response_cost"] == 0


@pytest.mark.asyncio
async def test_chunk_processor_logs_a_stream_that_ended_at_the_provider_error_frame_through_the_success_route():
    """A provider error frame that ends a stream nobody closed early is the request's final answer, and that
    stream keeps logging the way it always has: once, through the success route, with the usage it carried."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime.now(),
        litellm_call_id="test-final-error-frame",
        function_id="test-final-error-frame",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )
    frames = (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())
    received = [
        chunk
        async for chunk in PassThroughStreamingHandler.chunk_processor(
            response=_anthropic_stream_of(frames),
            request_body={"model": "claude-sonnet-5", "stream": True},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/v1/messages",
        )
    ]
    await _settle(recorder.success_kwargs)

    assert received == list(frames)
    assert recorder.failure_kwargs == []
    success_payload = recorder.success_kwargs[0]["standard_logging_object"]
    assert success_payload["status"] == "success"
    assert success_payload["prompt_tokens"] == 52
    assert logging_obj.model_call_details["prompt_cache_response_complete"] is False


@pytest.mark.asyncio
async def test_chunk_processor_bills_a_stream_the_client_closed_at_the_provider_error_frame():
    """The Anthropic SDK raises on a provider `event: error` frame and drops the connection, so a non-retried
    stream is closed at the frame by the client, not by a retry taking over. Its usage must still reach the
    success route, or the tokens the provider billed never reach the key's spend."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime.now(),
        litellm_call_id="test-client-closed-at-error-frame",
        function_id="test-client-closed-at-error-frame",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )
    stream = PassThroughStreamingHandler.chunk_processor(
        response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
        request_body={"model": "claude-sonnet-5", "stream": True},
        litellm_logging_obj=logging_obj,
        endpoint_type=EndpointType.ANTHROPIC,
        start_time=datetime.now(),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/v1/messages",
    )
    async for chunk in stream:
        if b"event: error" in chunk:
            break
    await stream.aclose()
    await _settle(recorder.success_kwargs)

    assert recorder.failure_kwargs == []
    success_payload = recorder.success_kwargs[0]["standard_logging_object"]
    assert success_payload["status"] == "success"
    assert success_payload["prompt_tokens"] == 52


@pytest.mark.asyncio
async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_retry_rebinds_the_logging_object():
    """The router rebinds the shared logging object to the retry's deployment right after closing the superseded
    stream, before the logging worker runs. The failure log of the superseded attempt must still name the
    deployment that served it, or the router's cooldown lands on the deployment serving the retry."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime.now(),
        litellm_call_id="test-superseded-deployment",
        function_id="test-superseded-deployment",
        dynamic_async_failure_callbacks=[recorder],
    )
    logging_obj.update_environment_variables(
        model="claude-sonnet-5", litellm_params={"model_info": {"id": "deployment-1"}}, optional_params={}
    )
    superseded = PassThroughStreamingHandler.chunk_processor(
        response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
        request_body={"model": "claude-sonnet-5", "stream": True},
        litellm_logging_obj=logging_obj,
        endpoint_type=EndpointType.ANTHROPIC,
        start_time=datetime.now(),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/v1/messages",
    )
    async for chunk in superseded:
        if b"event: error" in chunk:
            break
    await aclose_superseded_attempt(superseded)
    logging_obj.update_environment_variables(
        model="claude-sonnet-5-retry", litellm_params={"model_info": {"id": "deployment-2"}}, optional_params={}
    )
    await _settle(recorder.failure_kwargs)

    failure_kwargs = recorder.failure_kwargs[0]
    assert failure_kwargs["litellm_params"]["model_info"]["id"] == "deployment-1"
    assert failure_kwargs["model"] == "claude-sonnet-5"
    assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2"


@pytest.mark.asyncio
async def test_error_frame_then_exhausted_retry_bills_both_attempts() -> None:
    recorder: Final = _EventRecorder()
    logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-error-frame-terminal-billing")
    superseded: Final = _anthropic_messages_attempt(
        logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
    )
    async for chunk in superseded:
        if b"event: error" in chunk:
            break
    await aclose_superseded_attempt(superseded)
    carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj})
    await _drop_after_message_start(logging_obj, "msg_a2", 9)
    await _settle(recorder.failure_kwargs, count=2)

    failures: Final = tuple(event["standard_logging_object"] for event in recorder.failure_kwargs)
    assert tuple(event["prompt_tokens"] for event in failures) == (52, 9)
    terminal: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj})
    usage: Final = terminal["combined_usage_object"]
    assert isinstance(usage, litellm.Usage)
    assert usage.prompt_tokens == sum(event["prompt_tokens"] for event in failures)
    assert usage.completion_tokens == sum(event["completion_tokens"] for event in failures)
    assert terminal["response_cost"] == pytest.approx(sum(event["response_cost"] for event in failures))


@pytest.mark.asyncio
async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame():
    """Both attempts of a retried /v1/messages stream log through the request's one logging object. The
    superseded attempt's failure log must not leave the usage it consumed behind, or the retry's success
    log bills the failed attempt's tokens instead of its own."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="claude-sonnet-5",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="anthropic_messages",
        start_time=datetime.now(),
        litellm_call_id="test-retry-own-usage",
        function_id="test-retry-own-usage",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )

    def attempt(frames: tuple[bytes, ...]):
        return PassThroughStreamingHandler.chunk_processor(
            response=_anthropic_stream_of(frames),
            request_body={"model": "claude-sonnet-5", "stream": True},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.ANTHROPIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/v1/messages",
        )

    superseded = attempt((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
    async for chunk in superseded:
        if b"event: error" in chunk:
            break
    await aclose_superseded_attempt(superseded)
    carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj})
    await _settle(recorder.failure_kwargs)
    retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
    async for _ in retry:
        pass
    await _settle(recorder.success_kwargs)

    failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
    assert failure_payload["prompt_tokens"] == 52
    success_payload = recorder.success_kwargs[0]["standard_logging_object"]
    assert success_payload["status"] == "success"
    assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes:
    payload = {
        "candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}],
        "usageMetadata": {
            "promptTokenCount": prompt_tokens,
            "candidatesTokenCount": completion_tokens,
            "totalTokenCount": prompt_tokens + completion_tokens,
        },
        "modelVersion": "gemini-3.8-flash",
    }
    return f"data: {json.dumps(payload)}\r\n\r\n".encode()


def _google_stream_that_times_out_mid_stream():
    mock = MagicMock(spec=httpx.Response)
    mock.status_code = 200

    async def _aiter_bytes():
        yield _google_sse(9, 4, "The sea")
        yield _google_sse(9, 12, " is wide and restless")
        raise httpx.ReadTimeout("Timeout on reading data from socket")

    mock.aiter_bytes = _aiter_bytes
    return mock


@pytest.mark.parametrize(
    "endpoint_type, url_route",
    [
        (EndpointType.GEMINI, "/gemini/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse"),
        (
            EndpointType.VERTEX_AI,
            "/vertex_ai/v1/projects/p/locations/us-central1/publishers/google/models/gemini-3.8-flash:streamGenerateContent?alt=sse",
        ),
    ],
)
@pytest.mark.asyncio
async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exception(endpoint_type, url_route):
    """Google streams carry cumulative usage on every chunk, so a stream that
    dies mid-way must log a failure billed at what was already delivered rather
    than a failure at zero usage."""
    recorder = _EventRecorder()
    logging_obj = LiteLLMLoggingObj(
        model="gemini-3.8-flash",
        messages=[{"role": "user", "content": "hi"}],
        stream=True,
        call_type="pass_through_endpoint",
        start_time=datetime.now(),
        litellm_call_id=f"test-google-mid-stream-timeout-{endpoint_type.value}",
        function_id="test-google-mid-stream-timeout",
        dynamic_async_success_callbacks=[recorder],
        dynamic_async_failure_callbacks=[recorder],
    )
    logging_obj.update_environment_variables(
        model="gemini-3.8-flash",
        user="unknown",
        optional_params={},
        litellm_params={"metadata": {}},
        call_type="pass_through_endpoint",
    )
    success_routes = []

    async def _record_success_route(**kwargs):
        success_routes.append(kwargs)

    async def _consume_stream():
        async for _ in PassThroughStreamingHandler.chunk_processor(
            response=_google_stream_that_times_out_mid_stream(),
            request_body={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
            litellm_logging_obj=logging_obj,
            endpoint_type=endpoint_type,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route=url_route,
            route_streaming_logging=_record_success_route,
        ):
            pass

    with pytest.raises(httpx.ReadTimeout):
        await _consume_stream()

    for _ in range(300):
        if recorder.failure_kwargs:
            break
        await asyncio.sleep(0.01)

    assert success_routes == []
    assert recorder.success_kwargs == []
    assert len(recorder.failure_kwargs) == 1
    failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
    assert failure_payload["status"] == "failure"
    assert failure_payload["prompt_tokens"] == 9
    assert failure_payload["completion_tokens"] == 12
    assert failure_payload["response_cost"] > 12 * 3.75e-06
    assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)


class TestTinyFishStreamBilling:
    """SSE billing is owned by the detached poller spawned on the first run_id frame; it must
    survive gen.aclose() (client disconnect) and the stream-end path must not double-bill."""

    RUNS_URL = "https://agent.tinyfish.ai/v1/runs/run-sse-1?screenshots=none"
    SSE_ROUTE = "https://agent.tinyfish.ai/v1/automation/run-sse"

    @pytest.fixture
    def tinyfish_env(self, monkeypatch):
        monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test")
        monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False)
        monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False)
        # aiohttp transport bypasses respx; force plain httpx and drop any cached aiohttp client
        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
        litellm.in_memory_llm_clients_cache.flush_cache()

    def _tinyfish_logging_obj(self):
        obj = _unarmed_logging_obj()
        obj.model_call_details = {}
        obj.dispatch_success_handlers = AsyncMock()
        return obj

    def _spawned_since(self, tasks_before):
        return tinyfish_handler_module._BACKGROUND_BILLING_TASKS - tasks_before

    @pytest.mark.asyncio
    async def test_run_id_split_across_chunks_spawns_one_poller(self, tinyfish_env):
        chunks = [
            b'data: {"run_id": "run-s',
            b'se-1", "event": "INITIALIZED"}\n\n',
            b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n',
        ]
        logging_obj = self._tinyfish_logging_obj()
        tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)

        with respx.mock(assert_all_called=True) as upstream:
            upstream.get(self.RUNS_URL).respond(
                json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"}
            )
            received = []
            async for chunk in PassThroughStreamingHandler.chunk_processor(
                response=_make_streaming_response(chunks),
                request_body={"url": "https://scrapeme.live/shop", "goal": "extract"},
                litellm_logging_obj=logging_obj,
                endpoint_type=EndpointType.TINYFISH,
                start_time=datetime.now(),
                passthrough_success_handler_obj=MagicMock(),
                url_route=self.SSE_ROUTE,
                route_streaming_logging=AsyncMock(),
            ):
                received.append(chunk)

            assert received == chunks
            assert sse_poller_spawned(logging_obj)
            spawned = self._spawned_since(tasks_before)
            assert len(spawned) == 1
            await asyncio.gather(*spawned)

        logging_obj.dispatch_success_handlers.assert_awaited_once()
        assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032)

    @pytest.mark.asyncio
    async def test_poller_survives_client_disconnect_and_bills(self, tinyfish_env):
        chunks = [
            b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n',
            b'data: {"run_id": "run-sse-1", "event": "ACTION"}\n\n',
        ]
        logging_obj = self._tinyfish_logging_obj()
        tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)

        with respx.mock(assert_all_called=True) as upstream:
            upstream.get(self.RUNS_URL).respond(
                json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"}
            )
            gen = PassThroughStreamingHandler.chunk_processor(
                response=_make_streaming_response(chunks),
                request_body={"url": "https://scrapeme.live/shop", "goal": "extract"},
                litellm_logging_obj=logging_obj,
                endpoint_type=EndpointType.TINYFISH,
                start_time=datetime.now(),
                passthrough_success_handler_obj=MagicMock(),
                url_route=self.SSE_ROUTE,
                route_streaming_logging=AsyncMock(),
            )
            await gen.__anext__()
            spawned = self._spawned_since(tasks_before)
            assert len(spawned) == 1
            await gen.aclose()

            task = next(iter(spawned))
            assert not task.cancelled()
            await task

        logging_obj.dispatch_success_handlers.assert_awaited_once()
        assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032)

    @pytest.mark.asyncio
    async def test_stream_without_run_id_spawns_nothing(self, tinyfish_env):
        logging_obj = self._tinyfish_logging_obj()
        tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)

        async for _ in PassThroughStreamingHandler.chunk_processor(
            response=_make_streaming_response([b": keepalive\n\n", b'data: {"event": "HEARTBEAT"}\n\n']),
            request_body={},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.TINYFISH,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route=self.SSE_ROUTE,
            route_streaming_logging=AsyncMock(),
        ):
            pass

        assert not sse_poller_spawned(logging_obj)
        assert self._spawned_since(tasks_before) == set()

    @pytest.mark.asyncio
    async def test_stream_end_skips_dispatch_when_poller_owns_billing(self, tinyfish_env):
        logging_obj = self._tinyfish_logging_obj()
        mark_sse_poller_spawned(logging_obj)

        await PassThroughStreamingHandler._route_streaming_logging_to_handler(
            litellm_logging_obj=logging_obj,
            passthrough_success_handler_obj=MagicMock(),
            url_route=self.SSE_ROUTE,
            request_body={},
            endpoint_type=EndpointType.TINYFISH,
            start_time=datetime.now(),
            raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n'],
            end_time=datetime.now(),
        )

        logging_obj.dispatch_success_handlers.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_stream_end_fallback_still_logs_when_no_poller_spawned(self, tinyfish_env):
        logging_obj = self._tinyfish_logging_obj()

        await PassThroughStreamingHandler._route_streaming_logging_to_handler(
            litellm_logging_obj=logging_obj,
            passthrough_success_handler_obj=MagicMock(),
            url_route=self.SSE_ROUTE,
            request_body={},
            endpoint_type=EndpointType.TINYFISH,
            start_time=datetime.now(),
            raw_bytes=[b": keepalive\n\n"],
            end_time=datetime.now(),
        )

        logging_obj.dispatch_success_handlers.assert_awaited_once()
        assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None

    @pytest.mark.asyncio
    async def test_upstream_error_after_spawn_skips_failure_dispatch(self, tinyfish_env):
        logging_obj = self._tinyfish_logging_obj()
        logging_obj.dispatch_failure_handlers = MagicMock()
        tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)

        async def _aiter_bytes():
            yield b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n'
            raise httpx.ReadTimeout("upstream died")

        response = MagicMock(spec=httpx.Response)
        response.status_code = 200
        response.aiter_bytes = _aiter_bytes

        async def _consume():
            async for _ in PassThroughStreamingHandler.chunk_processor(
                response=response,
                request_body={},
                litellm_logging_obj=logging_obj,
                endpoint_type=EndpointType.TINYFISH,
                start_time=datetime.now(),
                passthrough_success_handler_obj=MagicMock(),
                url_route=self.SSE_ROUTE,
                route_streaming_logging=AsyncMock(),
            ):
                pass

        with pytest.raises(httpx.ReadTimeout):
            await _consume()

        spawned = self._spawned_since(tasks_before)
        assert len(spawned) == 1
        # the poller owns the single row; a failure dispatch would collide on its request_id
        logging_obj.dispatch_failure_handlers.assert_not_called()
        for task in spawned:
            task.cancel()

    @pytest.mark.asyncio
    async def test_unterminated_run_id_frame_late_spawns_poller(self, tinyfish_env):
        logging_obj = self._tinyfish_logging_obj()
        tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)

        await PassThroughStreamingHandler._route_streaming_logging_to_handler(
            litellm_logging_obj=logging_obj,
            passthrough_success_handler_obj=MagicMock(),
            url_route=self.SSE_ROUTE,
            request_body={},
            endpoint_type=EndpointType.TINYFISH,
            start_time=datetime.now(),
            raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}'],
            end_time=datetime.now(),
        )

        spawned = self._spawned_since(tasks_before)
        assert len(spawned) == 1
        assert sse_poller_spawned(logging_obj)
        # the poller polls to terminal instead of the old single fetch that mispriced a RUNNING run at $0
        logging_obj.dispatch_success_handlers.assert_not_awaited()
        for task in spawned:
            task.cancel()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "deferred_dispatch_armed",
    [False, True],
    ids=["enqueued-at-end-of-stream", "parked-for-deferred-dispatch"],
)
async def test_chunk_processor_claims_the_budget_reservation_before_handing_it_to_the_cost_callback(
    deferred_dispatch_armed: bool,
):
    reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False}
    response = _make_streaming_response([b"event-1", b"event-2"])
    logging_obj = _unarmed_logging_obj()
    logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}}
    if deferred_dispatch_armed:
        logging_obj._on_deferred_stream_complete = AsyncMock()
    claimed_when_the_callback_ran = []

    async def cost_callback(**kwargs):
        claimed_when_the_callback_ran.append(reservation["callback_bound"])

    async for _ in PassThroughStreamingHandler.chunk_processor(
        response=response,
        request_body={"model": "claude-3-haiku"},
        litellm_logging_obj=logging_obj,
        endpoint_type=EndpointType.GENERIC,
        start_time=datetime.now(),
        passthrough_success_handler_obj=MagicMock(),
        url_route="/bedrock/model/claude/invoke-with-response-stream",
        route_streaming_logging=cost_callback,
    ):
        pass

    if deferred_dispatch_armed:
        (parked_cost_callback,) = logging_obj._deferred_stream_complete_args
        await parked_cost_callback
    else:
        await GLOBAL_LOGGING_WORKER.flush()

    assert reservation["callback_bound"] is True
    assert claimed_when_the_callback_ran == [True]


@pytest.mark.asyncio
async def test_chunk_processor_leaves_the_budget_reservation_for_the_request_end_release_when_the_cost_callback_cannot_be_enqueued():
    reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False}
    response = _make_streaming_response([b"event-1", b"event-2"])
    logging_obj = _unarmed_logging_obj()
    logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}}

    def refuse_to_enqueue(async_coroutine):
        async_coroutine.close()
        raise RuntimeError("logging worker is shutting down")

    with patch.object(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", side_effect=refuse_to_enqueue):
        async for _ in PassThroughStreamingHandler.chunk_processor(
            response=response,
            request_body={"model": "claude-3-haiku"},
            litellm_logging_obj=logging_obj,
            endpoint_type=EndpointType.GENERIC,
            start_time=datetime.now(),
            passthrough_success_handler_obj=MagicMock(),
            url_route="/bedrock/model/claude/invoke-with-response-stream",
            route_streaming_logging=AsyncMock(),
        ):
            pass

    assert reservation["callback_bound"] is False