The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""Regression tests for PassThroughStreamingHandler.chunk_processor."""
import asyncio
import json
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from datetime import datetime
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
import litellm
import litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler as tinyfish_handler_module
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import (
mark_sse_poller_spawned,
sse_poller_spawned,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
from litellm.proxy.utils import _failure_fields_to_lift
from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
def _make_streaming_response(chunks):
mock = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes():
for c in chunks:
yield c
mock.aiter_bytes = _aiter_bytes
return mock
def _unarmed_logging_obj():
"""Real Logging objects only carry _on_deferred_stream_complete when the
proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
and would spuriously trigger the deferral branch."""
obj = MagicMock()
obj._on_deferred_stream_complete = None
return obj
@pytest.mark.asyncio
async def test_chunk_processor_logs_on_normal_completion():
chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
response = _make_streaming_response(chunks)
mock_logging_obj = _unarmed_logging_obj()
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
) as mock_route:
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/bedrock/model/claude/invoke-with-response-stream",
):
received.append(chunk)
await asyncio.sleep(0)
assert received == chunks
mock_route.assert_called_once()
call_kwargs = mock_route.call_args.kwargs
assert call_kwargs["raw_bytes"] == chunks
@pytest.mark.asyncio
async def test_chunk_processor_logs_on_client_disconnect():
chunks = [b"event-1", b"event-2", b"event-3"]
response = _make_streaming_response(chunks)
mock_logging_obj = _unarmed_logging_obj()
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
) as mock_route:
gen = PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/bedrock/model/claude/invoke-with-response-stream",
)
first = await gen.__anext__()
await gen.aclose()
await asyncio.sleep(0)
assert first == chunks[0]
mock_route.assert_called_once()
call_kwargs = mock_route.call_args.kwargs
assert call_kwargs["raw_bytes"] == [chunks[0]]
@pytest.mark.asyncio
async def test_chunk_processor_does_not_schedule_success_logging_for_upstream_error():
"""A 4xx/5xx upstream response is already logged as a failure by the caller
before this generator starts; scheduling success logging here too would
double-log the same request in SpendLogs."""
chunks = [b'{"error": "denied"}']
response = _make_streaming_response(chunks)
response.status_code = 403
mock_logging_obj = _unarmed_logging_obj()
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
) as mock_route:
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/bedrock/model/claude/invoke-with-response-stream",
):
received.append(chunk)
await asyncio.sleep(0)
assert received == chunks
mock_route.assert_not_called()
@pytest.mark.asyncio
async def test_chunk_processor_does_not_schedule_logging_when_no_chunks():
response = _make_streaming_response([])
mock_logging_obj = _unarmed_logging_obj()
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
) as mock_route:
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/bedrock/model/claude/invoke-with-response-stream",
):
received.append(chunk)
assert received == []
mock_route.assert_not_called()
@pytest.mark.asyncio
async def test_chunk_processor_routes_logging_through_logging_worker():
"""The spend-log coroutine must be handed to the durable logging worker, which
keeps a strong reference and drains on shutdown, instead of a bare
asyncio.create_task that the event loop only weak-references and can drop
under GC/load, silently losing the SpendLogs row for a successful call."""
chunks = [b"chunk-1", b"chunk-2"]
response = _make_streaming_response(chunks)
enqueued = []
def _capture(async_coroutine):
enqueued.append(async_coroutine)
async_coroutine.close()
with (
patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
),
patch.object(
GLOBAL_LOGGING_WORKER,
"ensure_initialized_and_enqueue",
side_effect=_capture,
) as mock_enqueue,
):
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=_unarmed_logging_obj(),
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/bedrock/model/claude/invoke-with-response-stream",
):
received.append(chunk)
assert received == chunks
mock_enqueue.assert_called_once()
assert asyncio.iscoroutine(enqueued[0])
@pytest.mark.asyncio
async def test_chunk_processor_routes_logging_through_logging_worker_on_disconnect():
"""Even when the client disconnects mid-stream, the partial-usage log must go
through the durable logging worker rather than a droppable bare task."""
chunks = [b"event-1", b"event-2", b"event-3"]
response = _make_streaming_response(chunks)
enqueued = []
def _capture(async_coroutine):
enqueued.append(async_coroutine)
async_coroutine.close()
with (
patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
),
patch.object(
GLOBAL_LOGGING_WORKER,
"ensure_initialized_and_enqueue",
side_effect=_capture,
) as mock_enqueue,
):
gen = PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=_unarmed_logging_obj(),
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/bedrock/model/claude/invoke-with-response-stream",
)
await gen.__anext__()
await gen.aclose()
mock_enqueue.assert_called_once()
assert asyncio.iscoroutine(enqueued[0])
def _logging_obj_with_write_once_cst():
"""Build a MagicMock that mirrors the real Logging behavior: update_completion_start_time
latches self.completion_start_time so the write-once guard actually latches."""
obj = _unarmed_logging_obj()
obj.completion_start_time = None
def _update(*, completion_start_time):
obj.completion_start_time = completion_start_time
obj.update_completion_start_time.side_effect = _update
return obj
@pytest.mark.asyncio
async def test_chunk_processor_stamps_completion_start_time_on_first_chunk():
"""Regression: LIT-4185 — streaming pass-through must stamp completion_start_time on
the first upstream chunk. Otherwise _success_handler_helper_fn falls back to
completion_start_time = end_time, and Prometheus/OTEL/SpendLogs TTFT reads as
total request duration (0 speedup)."""
chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
response = _make_streaming_response(chunks)
mock_logging_obj = _logging_obj_with_write_once_cst()
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
):
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-haiku-4-5"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/v1/messages",
):
received.append(chunk)
await asyncio.sleep(0)
assert received == chunks
mock_logging_obj.update_completion_start_time.assert_called_once()
stamped = mock_logging_obj.update_completion_start_time.call_args.kwargs["completion_start_time"]
assert isinstance(stamped, datetime)
@pytest.mark.asyncio
async def test_chunk_processor_does_not_reset_completion_start_time_on_later_chunks():
"""The stamp must be write-once: reading it on chunk 2/3 must not overwrite a real TTFT
from chunk 1 (which would collapse TTFT to time-to-last-chunk)."""
chunks = [b"chunk-1", b"chunk-2", b"chunk-3"]
response = _make_streaming_response(chunks)
real_first = datetime(2020, 1, 1, 0, 0, 0)
mock_logging_obj = _unarmed_logging_obj()
# Simulate first-chunk stamp having already landed (e.g. under contention or a
# prior wrapper that already set it): later chunks must be no-ops.
mock_logging_obj.completion_start_time = real_first
mock_passthrough_handler = MagicMock()
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
):
async for _ in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-haiku-4-5"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/v1/messages",
):
pass
mock_logging_obj.update_completion_start_time.assert_not_called()
assert mock_logging_obj.completion_start_time == real_first
@pytest.mark.asyncio
async def test_chunk_processor_stamps_completion_start_time_on_cost_injection_path():
"""The cost-injection branch runs alongside a hot path; both must stamp TTFT."""
import litellm as litellm_mod
chunks = [b"event: message_start\ndata: {}\n\n"]
response = _make_streaming_response(chunks)
mock_logging_obj = _logging_obj_with_write_once_cst()
mock_logging_obj.model_call_details = {"model": "claude-haiku-4-5"}
mock_passthrough_handler = MagicMock()
original = getattr(litellm_mod, "include_cost_in_streaming_usage", False)
litellm_mod.include_cost_in_streaming_usage = True
try:
with patch.object(
PassThroughStreamingHandler,
"route_streaming_logging_to_handler",
new=AsyncMock(),
):
async for _ in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-haiku-4-5"},
litellm_logging_obj=mock_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=mock_passthrough_handler,
url_route="/v1/messages",
):
pass
finally:
litellm_mod.include_cost_in_streaming_usage = original
mock_logging_obj.update_completion_start_time.assert_called_once()
def _openai_passthrough_stream_chunks():
return [
(
b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk",'
b'"choices":[{"index":0,"delta":{"content":"Hi"}}],"usage":null}\n\n'
),
b": keepalive\n\n",
(
b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[],'
b'"usage":{"prompt_tokens":11,"completion_tokens":4,"total_tokens":15,'
b'"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},'
b'"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,'
b'"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}\n\n'
),
b"data: [DONE]\n\n",
]
async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
response = _make_streaming_response(chunks)
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "gpt-4o-mini", "stream": True},
litellm_logging_obj=_unarmed_logging_obj(),
endpoint_type=endpoint_type,
start_time=datetime(2026, 1, 1),
passthrough_success_handler_obj=MagicMock(),
url_route="/openai/v1/chat/completions",
route_streaming_logging=AsyncMock(),
):
received.append(chunk)
await asyncio.sleep(0)
return received
@pytest.mark.asyncio
async def test_chunk_processor_injects_cost_into_openai_passthrough_usage_frame(monkeypatch):
"""Regression: issue #36492 — with include_cost_in_streaming_usage on, the final
OpenAI passthrough chat.completion.chunk usage frame must carry usage.cost, like
every other streaming surface already does."""
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
chunks = _openai_passthrough_stream_chunks()
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)
assert received[0] == chunks[0]
assert received[1] == chunks[1]
assert received[3] == chunks[3]
final_payload = json.loads(received[2].decode("utf-8").split("data:", 1)[1].strip())
pricing = litellm.model_cost["gpt-4o-mini"]
expected_cost = 11 * pricing["input_cost_per_token"] + 4 * pricing["output_cost_per_token"]
assert final_payload["usage"]["cost"] == pytest.approx(expected_cost)
assert final_payload["usage"]["cost"] > 0
assert final_payload["usage"]["prompt_tokens"] == 11
assert final_payload["usage"]["completion_tokens"] == 4
assert final_payload["usage"]["total_tokens"] == 15
@pytest.mark.asyncio
async def test_chunk_processor_injects_cost_into_usage_frame_fragmented_across_chunks(monkeypatch):
"""Regression: an SSE usage frame split across transport chunks must still get
cost injected once the frame completes, instead of passing through untouched."""
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
whole = _openai_passthrough_stream_chunks()
usage_frame = whole[2]
split_at = len(usage_frame) // 2
chunks = [whole[0], whole[1], usage_frame[:split_at], usage_frame[split_at:], whole[3]]
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)
reassembled = b"".join(received).decode("utf-8")
usage_lines = [ln for ln in reassembled.split("\n") if '"total_tokens"' in ln]
assert len(usage_lines) == 1
final_payload = json.loads(usage_lines[0].split("data:", 1)[1].strip())
assert final_payload["usage"]["cost"] > 0
assert final_payload["usage"]["prompt_tokens"] == 11
assert reassembled.endswith("data: [DONE]\n\n")
@pytest.mark.asyncio
async def test_chunk_processor_streams_crlf_delimited_frames_live_and_injects_cost(monkeypatch):
"""Regression: CRLF-delimited SSE frames must flow as they complete instead of
buffering until EOF, and the usage frame must still get cost injected."""
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
chunks = [chunk.replace(b"\n\n", b"\r\n\r\n") for chunk in _openai_passthrough_stream_chunks()]
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)
assert len(received) == len(chunks)
assert received[0] == chunks[0]
injected_usage_frame = received[2]
assert injected_usage_frame.endswith(b"\r\n\r\n")
assert b"\n" not in injected_usage_frame.replace(b"\r\n", b"")
reassembled = b"".join(received).decode("utf-8")
usage_lines = [ln for ln in reassembled.replace("\r\n", "\n").split("\n") if '"total_tokens"' in ln]
assert len(usage_lines) == 1
final_payload = json.loads(usage_lines[0].split("data:", 1)[1].strip())
assert final_payload["usage"]["cost"] > 0
@pytest.mark.asyncio
async def test_chunk_processor_flag_off_leaves_openai_passthrough_stream_byte_identical(monkeypatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
chunks = _openai_passthrough_stream_chunks()
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)
assert received == chunks
@pytest.mark.asyncio
async def test_chunk_processor_flag_on_leaves_openai_frames_without_usage_untouched(monkeypatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
chunks = [
(
b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk",'
b'"choices":[{"index":0,"delta":{"content":"Hi"}}],"usage":null}\n\n'
),
b": keepalive\n\n",
b"not json at all\n\n",
b"data: [DONE]\n\n",
]
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.OPENAI)
assert received == chunks
@pytest.mark.asyncio
async def test_chunk_processor_flag_on_leaves_generic_passthrough_untouched(monkeypatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
chunks = _openai_passthrough_stream_chunks()
received = await _collect_openai_passthrough_chunks(chunks, EndpointType.GENERIC)
assert received == chunks
def test_convert_raw_bytes_survives_truncated_multibyte_sequence():
"""A stream cut mid-multibyte-sequence (client disconnect) must still decode
via errors="replace" so the usage events already received are logged, instead
of raising UnicodeDecodeError and dropping the whole request from SpendLogs."""
# the 3-byte "☃" (E2 98 83) is cut after 2 bytes, leaving an invalid sequence
# that strict utf-8 decode would raise on, discarding the message_delta line too
truncated_codepoint = "☃".encode("utf-8")[:2]
raw_bytes = [
b'data: {"text": "' + truncated_codepoint,
b'\ndata: {"type": "message_delta"}\n',
]
lines = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
assert any('"type": "message_delta"' in line for line in lines)
@pytest.mark.asyncio
async def test_chunk_processor_defers_logging_until_fire_when_armed():
"""Regression for PR #38722: native /v1/messages streams route through
chunk_processor, which enqueued the spend log the moment the stream ended,
racing the guardrail end-of-stream scan and logging
guardrail_information as null. With deferred dispatch armed, the completed
stream must park the logging coroutine on logging_obj and only enqueue it
when ProxyLogging._fire_deferred_stream_logging fires after the scan."""
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.utils import ProxyLogging
chunks = [b"event-1", b"event-2"]
response = _make_streaming_response(chunks)
logging_obj = _unarmed_logging_obj()
logging_obj._deferred_stream_complete_args = None
enqueued = []
def _capture(async_coroutine):
enqueued.append(async_coroutine)
async_coroutine.close()
with patch.object( # test-quality-ok: GLOBAL_LOGGING_WORKER is a process-global singleton with no injection seam
GLOBAL_LOGGING_WORKER,
"ensure_initialized_and_enqueue",
side_effect=_capture,
) as mock_enqueue:
gen = PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
route_streaming_logging=AsyncMock(),
)
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
response=gen,
route_type="anthropic_messages",
user_api_key_dict=MagicMock(),
logging_obj=logging_obj,
)
received = []
async for chunk in gen:
received.append(chunk)
await asyncio.sleep(0)
assert received == chunks
mock_enqueue.assert_not_called()
parked = logging_obj._deferred_stream_complete_args
assert isinstance(parked, tuple) and len(parked) == 1
assert asyncio.iscoroutine(parked[0])
ProxyLogging._fire_deferred_stream_logging({"litellm_logging_obj": logging_obj})
await asyncio.sleep(0)
mock_enqueue.assert_called_once()
@pytest.mark.asyncio
async def test_chunk_processor_enqueues_immediately_on_disconnect_even_when_armed():
"""Client disconnects never reach _fire_deferred_stream_logging, so parking
the coroutine there would lose the partial-usage spend log (LIT-2642); the
disconnect path must keep enqueueing immediately."""
chunks = [b"event-1", b"event-2", b"event-3"]
response = _make_streaming_response(chunks)
logging_obj = _unarmed_logging_obj()
async def _armed_closure(logging_coroutine):
raise AssertionError("deferred closure must not fire on disconnect")
logging_obj._on_deferred_stream_complete = _armed_closure
logging_obj._deferred_stream_complete_args = None
enqueued = []
def _capture(async_coroutine):
enqueued.append(async_coroutine)
async_coroutine.close()
with patch.object( # test-quality-ok: GLOBAL_LOGGING_WORKER is a process-global singleton with no injection seam
GLOBAL_LOGGING_WORKER,
"ensure_initialized_and_enqueue",
side_effect=_capture,
) as mock_enqueue:
gen = PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
route_streaming_logging=AsyncMock(),
)
await gen.__anext__()
await gen.aclose()
mock_enqueue.assert_called_once()
assert logging_obj._deferred_stream_complete_args is None
class _EventRecorder(CustomLogger):
def __init__(self):
super().__init__()
self.failure_kwargs = []
self.success_kwargs = []
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
self.failure_kwargs.append(kwargs)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
self.success_kwargs.append(kwargs)
def _anthropic_sse(event: str, payload: dict) -> bytes:
return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode()
def _anthropic_stream_that_times_out_mid_stream():
mock = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes():
yield _anthropic_sse(
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 52, "output_tokens": 1},
},
},
)
yield _anthropic_sse(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
)
yield _anthropic_sse(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}},
)
raise httpx.ReadTimeout("Timeout on reading data from socket")
mock.aiter_bytes = _aiter_bytes
return mock
@pytest.mark.asyncio
async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception():
"""A stream that dies after the first chunks is a failed request: the failure
callbacks must fire once with the partial usage and cost, and the success
routing must never run for it."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime(2026, 1, 1),
litellm_call_id="test-mid-stream-timeout",
function_id="test-mid-stream-timeout",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
success_routes = []
async def _record_success_route(**kwargs):
success_routes.append(kwargs)
received = []
async def _consume_stream():
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_that_times_out_mid_stream(),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
route_streaming_logging=_record_success_route,
):
received.append(chunk)
with pytest.raises(httpx.ReadTimeout):
await _consume_stream()
for _ in range(300):
if recorder.failure_kwargs:
break
await asyncio.sleep(0.01)
assert len(received) == 3
assert success_routes == []
assert recorder.success_kwargs == []
assert len(recorder.failure_kwargs) == 1
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["status"] == "failure"
assert failure_payload["prompt_tokens"] == 52
assert failure_payload["response_cost"] > 0
assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)
def _anthropic_stream_dropping_after(frames: tuple[bytes, ...]) -> httpx.Response:
mock: Final = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes() -> AsyncIterator[bytes]:
for frame in frames:
yield frame
raise httpx.ReadError("upstream closed the connection")
mock.aiter_bytes = _aiter_bytes
return mock
def _anthropic_messages_attempt(
logging_obj: LiteLLMLoggingObj, response: httpx.Response
) -> AsyncGenerator[bytes, None]:
return PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime(2026, 1, 1),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
def _anthropic_messages_logging_obj(recorder: _EventRecorder, call_id: str) -> LiteLLMLoggingObj:
return LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime(2026, 1, 1),
litellm_call_id=call_id,
function_id=call_id,
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
async def _drop_after_message_start(logging_obj: LiteLLMLoggingObj, message_id: str, input_tokens: int) -> None:
dropped: Final = _anthropic_messages_attempt(
logging_obj, _anthropic_stream_dropping_after((_anthropic_message_start_frame(message_id, input_tokens),))
)
with pytest.raises(httpx.ReadError):
async for _ in dropped:
pass
@pytest.mark.asyncio
async def test_dropped_attempt_logs_its_failure_without_claiming_the_shared_logging_object() -> None:
"""The router retries a /v1/messages stream the provider drops after message_start on the same logging
object. The dropped attempt's failure log bills its own usage and leaves the shared object free: the
retry's success log carries the retry's usage and no stale exception, while the stash stays for the
failure row of a request that runs out of retries until the router opens the next attempt."""
recorder: Final = _EventRecorder()
logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-dropped-attempt")
await _drop_after_message_start(logging_obj, "msg_a1", 52)
await _settle(recorder.failure_kwargs)
assert recorder.failure_kwargs[0]["standard_logging_object"]["prompt_tokens"] == 52
assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadError)
assert logging_obj.model_call_details.get("has_logged_async_failure") is not True
assert "exception" not in logging_obj.model_call_details
stashed: Final = logging_obj.model_call_details.get("combined_usage_object")
assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 52
logging_obj.carry_failed_attempt_usage()
retry: Final = _anthropic_messages_attempt(
logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
)
async for _ in retry:
pass
await _settle(recorder.success_kwargs)
success_payload: Final = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
assert recorder.success_kwargs[0].get("exception") is None
assert len(recorder.failure_kwargs) == 1
@pytest.mark.asyncio
async def test_every_dropped_attempt_logs_its_own_failure() -> None:
"""Two attempts of one /v1/messages request dropped in a row each log a failure billing their own usage;
the first attempt's log must not swallow the second's, and the shared object ends up holding the last
attempt's usage for the request's failure row."""
recorder: Final = _EventRecorder()
logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-every-dropped-attempt")
await _drop_after_message_start(logging_obj, "msg_a1", 52)
await _settle(recorder.failure_kwargs)
logging_obj.carry_failed_attempt_usage()
await _drop_after_message_start(logging_obj, "msg_a2", 9)
await _settle(recorder.failure_kwargs, count=2)
assert [kwargs["standard_logging_object"]["prompt_tokens"] for kwargs in recorder.failure_kwargs] == [52, 9]
assert all(isinstance(kwargs["exception"], httpx.ReadError) for kwargs in recorder.failure_kwargs)
stashed: Final = logging_obj.model_call_details.get("combined_usage_object")
assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 9
def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes:
return _anthropic_sse(
"message_start",
{
"type": "message_start",
"message": {
"id": message_id,
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": input_tokens, "output_tokens": 1},
},
},
)
def _anthropic_completed_stream_tail() -> tuple[bytes, ...]:
return (
_anthropic_sse(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
),
_anthropic_sse(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
),
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
_anthropic_sse(
"message_delta",
{"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 3}},
),
_anthropic_sse("message_stop", {"type": "message_stop"}),
)
async def _settle(recorded: Sequence[Mapping[str, object]], count: int = 1) -> None:
for _ in range(300):
if len(recorded) >= count:
return
await asyncio.sleep(0.01)
def _anthropic_overloaded_error_frame() -> bytes:
return _anthropic_sse("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}})
def _anthropic_stream_of(frames: tuple[bytes, ...]):
mock = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes():
for frame in frames:
yield frame
mock.aiter_bytes = _aiter_bytes
return mock
@pytest.mark.parametrize(
"frames, billed",
[
pytest.param(
(_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()), True, id="error-after-message-start"
),
pytest.param((_anthropic_overloaded_error_frame(),), False, id="error-frame-only"),
],
)
@pytest.mark.asyncio
async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_closed_at_the_frame(frames, billed):
"""The router retries a /v1/messages stream whose provider sent `event: error` before any content by closing
it at the frame. That attempt never completed: it logs once as a failure carrying whatever usage the provider
had reported, and the request's one-shot success log stays free for the retry that shares the logging object."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-error-frame",
function_id="test-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
stream = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
received = []
async for chunk in stream:
received.append(chunk)
if b"event: error" in chunk:
break
await aclose_superseded_attempt(stream)
await _settle(recorder.failure_kwargs)
assert received == list(frames)
assert recorder.success_kwargs == []
assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True
assert logging_obj.should_run_logging("async_failure")
assert len(recorder.failure_kwargs) == 1
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["status"] == "failure"
assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
if billed:
assert failure_payload["prompt_tokens"] == 52
assert failure_payload["response_cost"] > 0
else:
assert failure_payload["response_cost"] == 0
@pytest.mark.asyncio
async def test_chunk_processor_logs_a_stream_that_ended_at_the_provider_error_frame_through_the_success_route():
"""A provider error frame that ends a stream nobody closed early is the request's final answer, and that
stream keeps logging the way it always has: once, through the success route, with the usage it carried."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-final-error-frame",
function_id="test-final-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
frames = (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())
received = [
chunk
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
]
await _settle(recorder.success_kwargs)
assert received == list(frames)
assert recorder.failure_kwargs == []
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert success_payload["prompt_tokens"] == 52
assert logging_obj.model_call_details["prompt_cache_response_complete"] is False
@pytest.mark.asyncio
async def test_chunk_processor_bills_a_stream_the_client_closed_at_the_provider_error_frame():
"""The Anthropic SDK raises on a provider `event: error` frame and drops the connection, so a non-retried
stream is closed at the frame by the client, not by a retry taking over. Its usage must still reach the
success route, or the tokens the provider billed never reach the key's spend."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-client-closed-at-error-frame",
function_id="test-client-closed-at-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
stream = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
async for chunk in stream:
if b"event: error" in chunk:
break
await stream.aclose()
await _settle(recorder.success_kwargs)
assert recorder.failure_kwargs == []
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert success_payload["prompt_tokens"] == 52
@pytest.mark.asyncio
async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_retry_rebinds_the_logging_object():
"""The router rebinds the shared logging object to the retry's deployment right after closing the superseded
stream, before the logging worker runs. The failure log of the superseded attempt must still name the
deployment that served it, or the router's cooldown lands on the deployment serving the retry."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-superseded-deployment",
function_id="test-superseded-deployment",
dynamic_async_failure_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model="claude-sonnet-5", litellm_params={"model_info": {"id": "deployment-1"}}, optional_params={}
)
superseded = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
async for chunk in superseded:
if b"event: error" in chunk:
break
await aclose_superseded_attempt(superseded)
logging_obj.update_environment_variables(
model="claude-sonnet-5-retry", litellm_params={"model_info": {"id": "deployment-2"}}, optional_params={}
)
await _settle(recorder.failure_kwargs)
failure_kwargs = recorder.failure_kwargs[0]
assert failure_kwargs["litellm_params"]["model_info"]["id"] == "deployment-1"
assert failure_kwargs["model"] == "claude-sonnet-5"
assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2"
@pytest.mark.asyncio
async def test_error_frame_then_exhausted_retry_bills_both_attempts() -> None:
recorder: Final = _EventRecorder()
logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-error-frame-terminal-billing")
superseded: Final = _anthropic_messages_attempt(
logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
)
async for chunk in superseded:
if b"event: error" in chunk:
break
await aclose_superseded_attempt(superseded)
carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj})
await _drop_after_message_start(logging_obj, "msg_a2", 9)
await _settle(recorder.failure_kwargs, count=2)
failures: Final = tuple(event["standard_logging_object"] for event in recorder.failure_kwargs)
assert tuple(event["prompt_tokens"] for event in failures) == (52, 9)
terminal: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj})
usage: Final = terminal["combined_usage_object"]
assert isinstance(usage, litellm.Usage)
assert usage.prompt_tokens == sum(event["prompt_tokens"] for event in failures)
assert usage.completion_tokens == sum(event["completion_tokens"] for event in failures)
assert terminal["response_cost"] == pytest.approx(sum(event["response_cost"] for event in failures))
@pytest.mark.asyncio
async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame():
"""Both attempts of a retried /v1/messages stream log through the request's one logging object. The
superseded attempt's failure log must not leave the usage it consumed behind, or the retry's success
log bills the failed attempt's tokens instead of its own."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-retry-own-usage",
function_id="test-retry-own-usage",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
def attempt(frames: tuple[bytes, ...]):
return PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
superseded = attempt((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
async for chunk in superseded:
if b"event: error" in chunk:
break
await aclose_superseded_attempt(superseded)
carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj})
await _settle(recorder.failure_kwargs)
retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
async for _ in retry:
pass
await _settle(recorder.success_kwargs)
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["prompt_tokens"] == 52
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes:
payload = {
"candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}],
"usageMetadata": {
"promptTokenCount": prompt_tokens,
"candidatesTokenCount": completion_tokens,
"totalTokenCount": prompt_tokens + completion_tokens,
},
"modelVersion": "gemini-3.8-flash",
}
return f"data: {json.dumps(payload)}\r\n\r\n".encode()
def _google_stream_that_times_out_mid_stream():
mock = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes():
yield _google_sse(9, 4, "The sea")
yield _google_sse(9, 12, " is wide and restless")
raise httpx.ReadTimeout("Timeout on reading data from socket")
mock.aiter_bytes = _aiter_bytes
return mock
@pytest.mark.parametrize(
"endpoint_type, url_route",
[
(EndpointType.GEMINI, "/gemini/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse"),
(
EndpointType.VERTEX_AI,
"/vertex_ai/v1/projects/p/locations/us-central1/publishers/google/models/gemini-3.8-flash:streamGenerateContent?alt=sse",
),
],
)
@pytest.mark.asyncio
async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exception(endpoint_type, url_route):
"""Google streams carry cumulative usage on every chunk, so a stream that
dies mid-way must log a failure billed at what was already delivered rather
than a failure at zero usage."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="gemini-3.8-flash",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="pass_through_endpoint",
start_time=datetime.now(),
litellm_call_id=f"test-google-mid-stream-timeout-{endpoint_type.value}",
function_id="test-google-mid-stream-timeout",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model="gemini-3.8-flash",
user="unknown",
optional_params={},
litellm_params={"metadata": {}},
call_type="pass_through_endpoint",
)
success_routes = []
async def _record_success_route(**kwargs):
success_routes.append(kwargs)
async def _consume_stream():
async for _ in PassThroughStreamingHandler.chunk_processor(
response=_google_stream_that_times_out_mid_stream(),
request_body={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route=url_route,
route_streaming_logging=_record_success_route,
):
pass
with pytest.raises(httpx.ReadTimeout):
await _consume_stream()
for _ in range(300):
if recorder.failure_kwargs:
break
await asyncio.sleep(0.01)
assert success_routes == []
assert recorder.success_kwargs == []
assert len(recorder.failure_kwargs) == 1
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["status"] == "failure"
assert failure_payload["prompt_tokens"] == 9
assert failure_payload["completion_tokens"] == 12
assert failure_payload["response_cost"] > 12 * 3.75e-06
assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)
class TestTinyFishStreamBilling:
"""SSE billing is owned by the detached poller spawned on the first run_id frame; it must
survive gen.aclose() (client disconnect) and the stream-end path must not double-bill."""
RUNS_URL = "https://agent.tinyfish.ai/v1/runs/run-sse-1?screenshots=none"
SSE_ROUTE = "https://agent.tinyfish.ai/v1/automation/run-sse"
@pytest.fixture
def tinyfish_env(self, monkeypatch):
monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test")
monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False)
monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False)
# aiohttp transport bypasses respx; force plain httpx and drop any cached aiohttp client
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
def _tinyfish_logging_obj(self):
obj = _unarmed_logging_obj()
obj.model_call_details = {}
obj.dispatch_success_handlers = AsyncMock()
return obj
def _spawned_since(self, tasks_before):
return tinyfish_handler_module._BACKGROUND_BILLING_TASKS - tasks_before
@pytest.mark.asyncio
async def test_run_id_split_across_chunks_spawns_one_poller(self, tinyfish_env):
chunks = [
b'data: {"run_id": "run-s',
b'se-1", "event": "INITIALIZED"}\n\n',
b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n',
]
logging_obj = self._tinyfish_logging_obj()
tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)
with respx.mock(assert_all_called=True) as upstream:
upstream.get(self.RUNS_URL).respond(
json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"}
)
received = []
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=_make_streaming_response(chunks),
request_body={"url": "https://scrapeme.live/shop", "goal": "extract"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
route_streaming_logging=AsyncMock(),
):
received.append(chunk)
assert received == chunks
assert sse_poller_spawned(logging_obj)
spawned = self._spawned_since(tasks_before)
assert len(spawned) == 1
await asyncio.gather(*spawned)
logging_obj.dispatch_success_handlers.assert_awaited_once()
assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032)
@pytest.mark.asyncio
async def test_poller_survives_client_disconnect_and_bills(self, tinyfish_env):
chunks = [
b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n',
b'data: {"run_id": "run-sse-1", "event": "ACTION"}\n\n',
]
logging_obj = self._tinyfish_logging_obj()
tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)
with respx.mock(assert_all_called=True) as upstream:
upstream.get(self.RUNS_URL).respond(
json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"}
)
gen = PassThroughStreamingHandler.chunk_processor(
response=_make_streaming_response(chunks),
request_body={"url": "https://scrapeme.live/shop", "goal": "extract"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
route_streaming_logging=AsyncMock(),
)
await gen.__anext__()
spawned = self._spawned_since(tasks_before)
assert len(spawned) == 1
await gen.aclose()
task = next(iter(spawned))
assert not task.cancelled()
await task
logging_obj.dispatch_success_handlers.assert_awaited_once()
assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032)
@pytest.mark.asyncio
async def test_stream_without_run_id_spawns_nothing(self, tinyfish_env):
logging_obj = self._tinyfish_logging_obj()
tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)
async for _ in PassThroughStreamingHandler.chunk_processor(
response=_make_streaming_response([b": keepalive\n\n", b'data: {"event": "HEARTBEAT"}\n\n']),
request_body={},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
route_streaming_logging=AsyncMock(),
):
pass
assert not sse_poller_spawned(logging_obj)
assert self._spawned_since(tasks_before) == set()
@pytest.mark.asyncio
async def test_stream_end_skips_dispatch_when_poller_owns_billing(self, tinyfish_env):
logging_obj = self._tinyfish_logging_obj()
mark_sse_poller_spawned(logging_obj)
await PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
request_body={},
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n'],
end_time=datetime.now(),
)
logging_obj.dispatch_success_handlers.assert_not_awaited()
@pytest.mark.asyncio
async def test_stream_end_fallback_still_logs_when_no_poller_spawned(self, tinyfish_env):
logging_obj = self._tinyfish_logging_obj()
await PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
request_body={},
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
raw_bytes=[b": keepalive\n\n"],
end_time=datetime.now(),
)
logging_obj.dispatch_success_handlers.assert_awaited_once()
assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None
@pytest.mark.asyncio
async def test_upstream_error_after_spawn_skips_failure_dispatch(self, tinyfish_env):
logging_obj = self._tinyfish_logging_obj()
logging_obj.dispatch_failure_handlers = MagicMock()
tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)
async def _aiter_bytes():
yield b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n'
raise httpx.ReadTimeout("upstream died")
response = MagicMock(spec=httpx.Response)
response.status_code = 200
response.aiter_bytes = _aiter_bytes
async def _consume():
async for _ in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
route_streaming_logging=AsyncMock(),
):
pass
with pytest.raises(httpx.ReadTimeout):
await _consume()
spawned = self._spawned_since(tasks_before)
assert len(spawned) == 1
# the poller owns the single row; a failure dispatch would collide on its request_id
logging_obj.dispatch_failure_handlers.assert_not_called()
for task in spawned:
task.cancel()
@pytest.mark.asyncio
async def test_unterminated_run_id_frame_late_spawns_poller(self, tinyfish_env):
logging_obj = self._tinyfish_logging_obj()
tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS)
await PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route=self.SSE_ROUTE,
request_body={},
endpoint_type=EndpointType.TINYFISH,
start_time=datetime.now(),
raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}'],
end_time=datetime.now(),
)
spawned = self._spawned_since(tasks_before)
assert len(spawned) == 1
assert sse_poller_spawned(logging_obj)
# the poller polls to terminal instead of the old single fetch that mispriced a RUNNING run at $0
logging_obj.dispatch_success_handlers.assert_not_awaited()
for task in spawned:
task.cancel()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"deferred_dispatch_armed",
[False, True],
ids=["enqueued-at-end-of-stream", "parked-for-deferred-dispatch"],
)
async def test_chunk_processor_claims_the_budget_reservation_before_handing_it_to_the_cost_callback(
deferred_dispatch_armed: bool,
):
reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False}
response = _make_streaming_response([b"event-1", b"event-2"])
logging_obj = _unarmed_logging_obj()
logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}}
if deferred_dispatch_armed:
logging_obj._on_deferred_stream_complete = AsyncMock()
claimed_when_the_callback_ran = []
async def cost_callback(**kwargs):
claimed_when_the_callback_ran.append(reservation["callback_bound"])
async for _ in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/bedrock/model/claude/invoke-with-response-stream",
route_streaming_logging=cost_callback,
):
pass
if deferred_dispatch_armed:
(parked_cost_callback,) = logging_obj._deferred_stream_complete_args
await parked_cost_callback
else:
await GLOBAL_LOGGING_WORKER.flush()
assert reservation["callback_bound"] is True
assert claimed_when_the_callback_ran == [True]
@pytest.mark.asyncio
async def test_chunk_processor_leaves_the_budget_reservation_for_the_request_end_release_when_the_cost_callback_cannot_be_enqueued():
reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False}
response = _make_streaming_response([b"event-1", b"event-2"])
logging_obj = _unarmed_logging_obj()
logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}}
def refuse_to_enqueue(async_coroutine):
async_coroutine.close()
raise RuntimeError("logging worker is shutting down")
with patch.object(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", side_effect=refuse_to_enqueue):
async for _ in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "claude-3-haiku"},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/bedrock/model/claude/invoke-with-response-stream",
route_streaming_logging=AsyncMock(),
):
pass
assert reservation["callback_bound"] is False