The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import base64
import inspect
import json
import logging
import threading
import time
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
import respx
from botocore.credentials import RefreshableCredentials
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.code_interpreter_interception.handler import (
CodeInterpreterInterceptionLogger,
LITELLM_CODE_EXECUTION_TOOL_NAME,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse
from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS
from litellm.llms.brave.search.transformation import BraveSearchConfig
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.base_llm.image_generation.transformation import BaseImageGenerationConfig
from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig
from litellm.llms.chatgpt.authenticator import Authenticator as ChatGPTAuthenticator
from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import (
BaseLLMHTTPHandler,
_collect_ws_project_quota_callbacks,
_google_genai_streaming_hidden_params,
_has_pre_call_deployment_hook,
_rust_responses_websocket_enabled,
)
from litellm.llms.azure.videos.transformation import AzureVideoConfig
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
from litellm.llms.anthropic.skills.transformation import AnthropicSkillsConfig
from litellm.llms.openai.evals.transformation import OpenAIEvalsConfig
from litellm.llms.mistral.files.transformation import MistralFilesConfig
from litellm.llms.openai.vector_store_files.transformation import OpenAIVectorStoreFilesConfig
from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig
from litellm.llms.openai.videos.transformation import OpenAIVideoConfig
from litellm.llms.tinyfish.search.transformation import TinyfishSearchConfig
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
MockResponsesAPIStreamingIterator,
)
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
_ACTIVE_KEY = "_code_interpreter_interception_active"
_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
async def _get_search_with_client(
client: HTTPHandler | AsyncHTTPHandler, provider_config: BaseSearchConfig | None = None
) -> SearchResponse:
result: Final = BaseLLMHTTPHandler().search(
query="test",
optional_params={},
timeout=5,
logging_obj=Mock(),
api_key="test-key",
api_base="https://search.example.test/",
custom_llm_provider="tinyfish" if isinstance(provider_config, TinyfishSearchConfig) else "brave",
client=client,
asearch=isinstance(client, AsyncHTTPHandler),
provider_config=provider_config or BraveSearchConfig(),
)
return await result if asyncio.iscoroutine(result) else result
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.parametrize("status_code", (400, 401, 403, 422, 429, 500))
async def test_get_search_raises_provider_http_errors(is_async: bool, status_code: int) -> None:
upstream_response: Final = httpx.Response(
status_code, json={"error": "rejected request"}, headers={"retry-after": "7"}
)
transport: Final = httpx.MockTransport(lambda request: upstream_response)
async with httpx.AsyncClient(transport=transport) as async_client:
with httpx.Client(transport=transport) as sync_client:
client: Final = AsyncHTTPHandler() if is_async else HTTPHandler(client=sync_client)
if isinstance(client, AsyncHTTPHandler):
await client.close()
client.client = async_client
with pytest.raises(BaseLLMException) as error:
await _get_search_with_client(client)
assert error.value.status_code == status_code
assert "rejected request" in error.value.message
assert error.value.headers is not None
assert error.value.headers["retry-after"] == "7"
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.parametrize("has_results", (False, True))
async def test_get_search_preserves_successful_results(is_async: bool, has_results: bool) -> None:
results: Final = (
[{"title": "Example", "url": "https://example.com", "description": "Example snippet"}] if has_results else []
)
transport: Final = httpx.MockTransport(lambda request: httpx.Response(200, json={"web": {"results": results}}))
async with httpx.AsyncClient(transport=transport) as async_client:
with httpx.Client(transport=transport) as sync_client:
client: Final = AsyncHTTPHandler() if is_async else HTTPHandler(client=sync_client)
if isinstance(client, AsyncHTTPHandler):
await client.close()
client.client = async_client
response: Final = await _get_search_with_client(client)
assert response.object == "search"
assert len(response.results) == int(has_results)
if has_results:
assert response.results[0].title == "Example"
assert response.results[0].url == "https://example.com"
assert response.results[0].snippet == "Example snippet"
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", (False, True))
async def test_get_search_preserves_tinyfish_http_error_formatting(is_async: bool) -> None:
upstream_response: Final = httpx.Response(
429,
json={"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "rate limit exceeded"}},
headers={"retry-after": "7"},
)
transport: Final = httpx.MockTransport(lambda request: upstream_response)
async with httpx.AsyncClient(transport=transport) as async_client:
with httpx.Client(transport=transport) as sync_client:
client: Final = AsyncHTTPHandler() if is_async else HTTPHandler(client=sync_client)
if isinstance(client, AsyncHTTPHandler):
await client.close()
client.client = async_client
with pytest.raises(BaseLLMException) as error:
await _get_search_with_client(client, TinyfishSearchConfig())
assert error.value.status_code == 429
assert error.value.message == (
"TinyFish Search: rate limit exceeded. See https://docs.tinyfish.ai/search-api for details."
)
assert error.value.headers is not None
assert error.value.headers["retry-after"] == "7"
def test_prepare_fake_stream_request():
# Initialize the BaseLLMHTTPHandler
handler = BaseLLMHTTPHandler()
# Test case 1: fake_stream is True
stream = True
data = {
"stream": True,
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
}
fake_stream = True
result_stream, result_data = handler._prepare_fake_stream_request(stream=stream, data=data, fake_stream=fake_stream)
# Verify that stream is set to False
assert result_stream is False
# Verify that "stream" key is removed from data
assert "stream" not in result_data
# Verify other data remains unchanged
assert result_data["model"] == "gpt-4"
assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
# Test case 2: fake_stream is False
stream = True
data = {
"stream": True,
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
}
fake_stream = False
result_stream, result_data = handler._prepare_fake_stream_request(stream=stream, data=data, fake_stream=fake_stream)
# Verify that stream remains True
assert result_stream is True
# Verify that data remains unchanged
assert "stream" in result_data
assert result_data["stream"] is True
assert result_data["model"] == "gpt-4"
assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
# Test case 3: data doesn't have stream key but fake_stream is True
stream = True
data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}
fake_stream = True
result_stream, result_data = handler._prepare_fake_stream_request(stream=stream, data=data, fake_stream=fake_stream)
# Verify that stream is set to False
assert result_stream is False
# Verify that data remains unchanged (since there was no stream key to remove)
assert "stream" not in result_data
assert result_data["model"] == "gpt-4"
assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
def test_response_api_handler_streams_when_provider_transform_adds_stream():
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://chatgpt.example.com/responses"
config.transform_responses_api_request.return_value = {
"model": "gpt-5.3-codex",
"input": "hi",
"stream": True,
}
config.sign_request.return_value = ({}, None)
client = HTTPHandler(client=httpx.Client())
client.post = Mock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
)
logging_obj = Mock()
handler.response_api_handler(
model="gpt-5.3-codex",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={},
custom_llm_provider="chatgpt",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
client=client,
)
assert client.post.call_args.kwargs["stream"] is True
assert client.post.call_args.kwargs["json"]["stream"] is True
def test_response_api_handler_runs_agentic_hooks_in_sync_path(monkeypatch):
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://chatgpt.example.com/responses"
config.transform_responses_api_request.return_value = {
"model": "gpt-5",
"input": "hi",
}
config.sign_request.return_value = ({}, None)
initial_response = Mock()
final_response = Mock()
config.transform_response_api_response.return_value = initial_response
client = HTTPHandler(client=httpx.Client())
client.post = Mock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
)
logging_obj = Mock()
monkeypatch.setattr(handler, "_has_agentic_completion_hook", Mock(return_value=True))
hook_mock = AsyncMock(return_value=final_response)
monkeypatch.setattr(handler, "_call_agentic_completion_hooks", hook_mock)
response = handler.response_api_handler(
model="gpt-5",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={},
custom_llm_provider="openai",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
client=client,
)
assert response is final_response
hook_mock.assert_awaited_once()
assert hook_mock.call_args.kwargs["api_surface"] == "responses"
assert hook_mock.call_args.kwargs["messages"] == [{"role": "user", "content": "hi"}]
def test_response_api_handler_runs_responses_pre_call_hook_before_transform():
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://api.openai.com/v1/responses"
config.sign_request.return_value = ({}, None)
initial_response = ResponsesAPIResponse(
id="resp_1",
created_at=0,
output=[],
status="completed",
model="gpt-5",
)
config.transform_response_api_response.return_value = initial_response
def transform_responses_api_request(**kwargs):
return {
"model": kwargs["model"],
"input": kwargs["input"],
**kwargs["response_api_optional_request_params"],
}
config.transform_responses_api_request.side_effect = transform_responses_api_request
client = HTTPHandler(client=httpx.Client())
client.post = Mock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
)
)
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = []
old_callbacks = list(litellm.callbacks)
litellm.callbacks = [CodeInterpreterInterceptionLogger()]
try:
response = handler.response_api_handler(
model="gpt-5",
input="use code",
responses_api_provider_config=config,
response_api_optional_request_params={
"tools": [{"type": "code_interpreter", "container": {"type": "auto"}}]
},
custom_llm_provider="openai",
litellm_params=GenericLiteLLMParams(api_key="sk-test"),
logging_obj=logging_obj,
client=client,
)
finally:
litellm.callbacks = old_callbacks
assert response is initial_response
transform_kwargs = config.transform_responses_api_request.call_args.kwargs
tools = transform_kwargs["response_api_optional_request_params"]["tools"]
assert not any(tool.get("type") == "code_interpreter" for tool in tools)
assert any(
tool.get("type") == "function" and tool.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME for tool in tools
)
hook_litellm_params = transform_kwargs["litellm_params"]
assert hook_litellm_params.get(_ACTIVE_KEY) is True
assert hook_litellm_params.get(_SANDBOX_KEY)
class _AgenticLoopKwargsRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.seen_kwargs: Mapping[str, object] = MappingProxyType({})
async def async_should_run_agentic_loop(
self,
response: object,
model: str,
messages: list[dict[str, object]],
tools: list[dict[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict[str, object],
) -> tuple[bool, dict[str, object]]:
self.seen_kwargs = MappingProxyType(dict(kwargs))
return False, {}
class _SentRequestRecorder:
def __init__(self) -> None:
self.body: Mapping[str, object] = MappingProxyType({})
def respond(self, request: httpx.Request) -> httpx.Response:
self.body = MappingProxyType(json.loads(request.content))
return httpx.Response(
200,
json={
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-4o",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "hello", "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
)
def test_response_api_handler_sends_no_stream_options_when_web_search_forces_a_non_streaming_call(
monkeypatch: pytest.MonkeyPatch,
):
sent: Final = _SentRequestRecorder()
recorder: Final = _AgenticLoopKwargsRecorder()
monkeypatch.setattr(
litellm, "callbacks", [WebSearchInterceptionLogger(enabled_providers=["azure"]), recorder]
)
logging_obj: Final = Mock()
logging_obj.dynamic_success_callbacks = []
BaseLLMHTTPHandler().response_api_handler(
model="gpt-4o",
input="what is new in litellm?",
responses_api_provider_config=AzureOpenAIResponsesAPIConfig(),
response_api_optional_request_params={
"stream": True,
"stream_options": {"include_obfuscation": True},
"tools": [{"type": "web_search"}],
},
custom_llm_provider="azure",
litellm_params=GenericLiteLLMParams(
api_key="sk-test",
api_base="https://example.openai.azure.com",
api_version="2025-04-01-preview",
stream_options={"include_obfuscation": True},
),
logging_obj=logging_obj,
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(sent.respond))),
)
assert sent.body["stream"] is False
assert "stream_options" not in sent.body
assert "stream_options" not in recorder.seen_kwargs
@pytest.mark.asyncio
async def test_async_response_api_handler_streams_when_provider_transform_adds_stream():
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://chatgpt.example.com/responses"
config.async_transform_responses_api_request = AsyncMock(
return_value={"model": "gpt-5.3-codex", "input": "hi", "stream": True}
)
config.sign_request.return_value = ({}, None)
client = AsyncHTTPHandler()
client.post = AsyncMock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
)
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = None
await handler.async_response_api_handler(
model="gpt-5.3-codex",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={},
custom_llm_provider="chatgpt",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
client=client,
)
assert client.post.call_args.kwargs["stream"] is True
assert client.post.call_args.kwargs["json"]["stream"] is True
@pytest.mark.asyncio
async def test_async_response_api_handler_validates_environment_off_the_event_loop() -> None:
def validate_environment(**_kwargs: object) -> dict[str, str]:
return {"validated-on-thread": str(threading.get_ident()), "x-override": "provider"}
config: Final = Mock()
config.validate_environment.side_effect = validate_environment
config.get_complete_url.return_value = "https://responses.example.com/v1/responses"
config.async_transform_responses_api_request = AsyncMock(return_value={"model": "model", "input": "hi"})
config.sign_request.return_value = ({}, None)
client: Final = AsyncMock(spec=AsyncHTTPHandler)
client.post.return_value = httpx.Response(
200,
request=httpx.Request("POST", "https://responses.example.com/v1/responses"),
)
logging_obj: Final = Mock()
logging_obj.dynamic_success_callbacks = None
await BaseLLMHTTPHandler().async_response_api_handler(
model="model",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={},
custom_llm_provider="test",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
client=client,
extra_headers={"x-override": "caller"},
)
validated_headers: Final = config.async_transform_responses_api_request.await_args.kwargs["headers"]
assert validated_headers["validated-on-thread"] != str(threading.get_ident())
assert validated_headers["x-override"] == "caller"
client.post.assert_awaited_once()
_CHATGPT_SSE_BODY = (
"event: response.output_item.done\n"
'data: {"type": "response.output_item.done", "output_index": 0, "item": {"type": "message", '
'"id": "msg_1", "status": "completed", "role": "assistant", "content": [{"type": "output_text", '
'"text": "aggregated", "annotations": []}]}}\n'
"\n"
"event: response.completed\n"
'data: {"type": "response.completed", "response": {"id": "resp_1", "object": "response", '
'"created_at": 1, "model": "gpt-5.3-codex", "status": "completed", "output": [], '
'"parallel_tool_calls": false, "tool_choice": "auto", "tools": [], '
'"usage": {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}}\n'
"\n"
)
def _chatgpt_sse_response():
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content=_CHATGPT_SSE_BODY.encode(),
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
def _chatgpt_responses_logging_obj():
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = None
logging_obj.async_success_handler = AsyncMock()
return logging_obj
def _chatgpt_responses_config():
authenticator = Mock(spec=ChatGPTAuthenticator)
authenticator.get_access_token.return_value = "access-test"
authenticator.get_account_id.return_value = "acct-test"
return ChatGPTResponsesAPIConfig(authenticator=authenticator)
def _chatgpt_handler_kwargs(caller_params, client):
return {
"model": "gpt-5.3-codex",
"input": "hi",
"responses_api_provider_config": _chatgpt_responses_config(),
"response_api_optional_request_params": caller_params,
"custom_llm_provider": "chatgpt",
"litellm_params": GenericLiteLLMParams(api_key="sk-test", api_base="https://chatgpt.example.com"),
"logging_obj": _chatgpt_responses_logging_obj(),
"client": client,
}
def _assert_aggregated_chatgpt_response(result):
assert not isinstance(result, BaseResponsesAPIStreamingIterator)
assert isinstance(result, ResponsesAPIResponse)
assert result.id == "resp_1"
assert result.output[0].content[0].text == "aggregated"
@pytest.mark.parametrize("caller_params", [{}, {"stream": False}])
def test_response_api_handler_aggregates_chatgpt_sse_for_a_non_streaming_caller(caller_params):
handler = BaseLLMHTTPHandler()
client = HTTPHandler(client=httpx.Client())
client.post = Mock(return_value=_chatgpt_sse_response())
result = handler.response_api_handler(**_chatgpt_handler_kwargs(caller_params, client))
assert client.post.call_args.kwargs["json"]["stream"] is True
_assert_aggregated_chatgpt_response(result)
def test_response_api_handler_streams_chatgpt_sse_for_a_streaming_caller():
handler = BaseLLMHTTPHandler()
client = HTTPHandler(client=httpx.Client())
client.post = Mock(return_value=_chatgpt_sse_response())
result = handler.response_api_handler(**_chatgpt_handler_kwargs({"stream": True}, client))
assert isinstance(result, BaseResponsesAPIStreamingIterator)
assert [event.type for event in result][-1] == "response.completed"
def test_response_api_handler_streams_chatgpt_sse_for_an_extra_body_streaming_caller():
handler = BaseLLMHTTPHandler()
client = HTTPHandler(client=httpx.Client())
client.post = Mock(return_value=_chatgpt_sse_response())
result = handler.response_api_handler(extra_body={"stream": True}, **_chatgpt_handler_kwargs({}, client))
assert isinstance(result, BaseResponsesAPIStreamingIterator)
assert [event.type for event in result][-1] == "response.completed"
def test_response_api_handler_fake_streams_only_for_a_streaming_caller():
handler = BaseLLMHTTPHandler()
client = HTTPHandler(client=httpx.Client())
client.post = Mock(return_value=_chatgpt_sse_response())
streamed = handler.response_api_handler(fake_stream=True, **_chatgpt_handler_kwargs({"stream": True}, client))
assert isinstance(streamed, MockResponsesAPIStreamingIterator)
client.post = Mock(return_value=_chatgpt_sse_response())
aggregated = handler.response_api_handler(fake_stream=True, **_chatgpt_handler_kwargs({}, client))
_assert_aggregated_chatgpt_response(aggregated)
@pytest.mark.asyncio
@pytest.mark.parametrize("caller_params", [{}, {"stream": False}])
async def test_async_response_api_handler_aggregates_chatgpt_sse_for_a_non_streaming_caller(caller_params):
handler = BaseLLMHTTPHandler()
client = AsyncHTTPHandler()
client.post = AsyncMock(return_value=_chatgpt_sse_response())
result = await handler.async_response_api_handler(**_chatgpt_handler_kwargs(caller_params, client))
assert client.post.call_args.kwargs["json"]["stream"] is True
_assert_aggregated_chatgpt_response(result)
@pytest.mark.asyncio
async def test_async_response_api_handler_streams_chatgpt_sse_for_a_streaming_caller():
handler = BaseLLMHTTPHandler()
client = AsyncHTTPHandler()
client.post = AsyncMock(return_value=_chatgpt_sse_response())
result = await handler.async_response_api_handler(**_chatgpt_handler_kwargs({"stream": True}, client))
assert isinstance(result, BaseResponsesAPIStreamingIterator)
assert [event.type async for event in result][-1] == "response.completed"
@pytest.mark.asyncio
async def test_async_response_api_handler_streams_chatgpt_sse_for_an_extra_body_streaming_caller():
handler = BaseLLMHTTPHandler()
client = AsyncHTTPHandler()
client.post = AsyncMock(return_value=_chatgpt_sse_response())
result = await handler.async_response_api_handler(
extra_body={"stream": True}, **_chatgpt_handler_kwargs({}, client)
)
assert isinstance(result, BaseResponsesAPIStreamingIterator)
assert [event.type async for event in result][-1] == "response.completed"
@pytest.mark.asyncio
async def test_async_response_api_handler_fake_streams_only_for_a_streaming_caller():
handler = BaseLLMHTTPHandler()
client = AsyncHTTPHandler()
client.post = AsyncMock(return_value=_chatgpt_sse_response())
streamed = await handler.async_response_api_handler(
fake_stream=True, **_chatgpt_handler_kwargs({"stream": True}, client)
)
assert isinstance(streamed, MockResponsesAPIStreamingIterator)
client.post = AsyncMock(return_value=_chatgpt_sse_response())
aggregated = await handler.async_response_api_handler(fake_stream=True, **_chatgpt_handler_kwargs({}, client))
_assert_aggregated_chatgpt_response(aggregated)
@pytest.mark.asyncio
async def test_async_response_api_handler_streaming_passes_logging_obj_to_post():
"""LIT-5466: @track_llm_api_timing only records llm_api_duration_ms when the POST
receives logging_obj; without it streaming /v1/responses never gets
x-litellm-overhead-duration-ms (the non-streaming site is pinned by
test_async_responses_records_llm_api_duration below)."""
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://chatgpt.example.com/responses"
config.async_transform_responses_api_request = AsyncMock(
return_value={"model": "gpt-5", "input": "hi", "stream": True}
)
config.sign_request.return_value = ({}, None)
client = AsyncHTTPHandler()
client.post = AsyncMock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
)
logging_obj = Mock()
await handler.async_response_api_handler(
model="gpt-5",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={"stream": True},
custom_llm_provider="chatgpt",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
client=client,
)
assert client.post.call_args.kwargs["logging_obj"] is logging_obj
@pytest.mark.asyncio
async def test_async_response_api_handler_posts_the_async_transform_hook_result():
"""A provider whose request transform must await (Bedrock inlines remote image URLs)
overrides the async hook; the async handler has to send that result, not the sync one."""
handler = BaseLLMHTTPHandler()
config = Mock()
config.validate_environment.return_value = {}
config.get_complete_url.return_value = "https://chatgpt.example.com/responses"
config.async_transform_responses_api_request = AsyncMock(
return_value={"model": "gpt-5", "input": "inlined by the async hook", "stream": True}
)
config.sign_request.return_value = ({}, None)
client = AsyncHTTPHandler()
client.post = AsyncMock(
return_value=httpx.Response(
200,
request=httpx.Request("POST", "https://chatgpt.example.com/responses"),
)
)
await handler.async_response_api_handler(
model="gpt-5",
input="hi",
responses_api_provider_config=config,
response_api_optional_request_params={"stream": True},
custom_llm_provider="chatgpt",
litellm_params=GenericLiteLLMParams(),
logging_obj=Mock(),
client=client,
)
assert client.post.call_args.kwargs["json"]["input"] == "inlined by the async hook"
config.transform_responses_api_request.assert_not_called()
@pytest.mark.asyncio
async def test_async_responses_records_llm_api_duration():
"""aresponses must feed the httpx timing into the logging obj, so the proxy can emit
x-litellm-overhead-duration-ms on /v1/responses (mirrors the arerank regression test)."""
def handle(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "resp_1",
"object": "response",
"created_at": 1,
"model": "gpt-4o-mini",
"status": "completed",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
},
)
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
response = await litellm.aresponses(
model="openai/gpt-4o-mini",
input="ping",
api_key="fake-key",
client=client,
)
assert response._hidden_params["litellm_overhead_time_ms"] is not None
assert response._hidden_params["_response_ms"] >= response._hidden_params["litellm_overhead_time_ms"]
def test_get_agentic_loop_settings_defaults_and_overrides():
handler = BaseLLMHTTPHandler()
depth, max_loops, fingerprints = handler._get_agentic_loop_settings(kwargs={})
assert depth == 0
assert max_loops == 3
assert fingerprints == []
depth, max_loops, fingerprints = handler._get_agentic_loop_settings(
kwargs={
"_agentic_loop_depth": 2,
"max_agentic_loops": 7,
"_agentic_loop_fingerprints": ["fp-1", "fp-2"],
}
)
assert depth == 2
assert max_loops == 7
assert fingerprints == ["fp-1", "fp-2"]
def test_has_agentic_completion_hook_detection(monkeypatch):
"""The streaming path skips the agentic wrapper only when no callback
overrides async_should_run_agentic_loop. Verify both directions."""
from litellm.integrations.custom_logger import CustomLogger
handler = BaseLLMHTTPHandler()
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = []
# No callbacks at all -> no agentic hook.
monkeypatch.setattr(litellm, "callbacks", [])
assert handler._has_agentic_completion_hook(logging_obj) is False
# A plain CustomLogger that does NOT override the gate -> still no hook
# (so the wrapper is safely skipped).
class _PlainLogger(CustomLogger):
pass
monkeypatch.setattr(litellm, "callbacks", [_PlainLogger()])
assert handler._has_agentic_completion_hook(logging_obj) is False
# A logger that overrides the gate (directly) -> hook present.
class _AgenticLogger(CustomLogger):
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
return True, {}
monkeypatch.setattr(litellm, "callbacks", [_AgenticLogger()])
assert handler._has_agentic_completion_hook(logging_obj) is True
# Override inherited through an intermediate class is still detected
# (function-identity check, not a leaf __dict__ check).
class _DerivedAgenticLogger(_AgenticLogger):
pass
monkeypatch.setattr(litellm, "callbacks", [_DerivedAgenticLogger()])
assert handler._has_agentic_completion_hook(logging_obj) is True
# Hook supplied via logging_obj.dynamic_success_callbacks is detected too.
monkeypatch.setattr(litellm, "callbacks", [])
logging_obj.dynamic_success_callbacks = [_AgenticLogger()]
assert handler._has_agentic_completion_hook(logging_obj) is True
# String-named callback entry (e.g. "datadog") must be resolved to its
# CustomLogger instance via get_custom_logger_compatible_class -- the same
# way ProxyLogging._callback_capabilities handles them. Without that
# resolution a string-registered agentic callback would be silently
# skipped and the buffering wrapper would never fire.
logging_obj.dynamic_success_callbacks = []
agentic_via_string = _AgenticLogger()
monkeypatch.setattr(litellm, "callbacks", ["fake_string_callback"])
monkeypatch.setattr(
"litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class",
lambda name: agentic_via_string if name == "fake_string_callback" else None,
)
assert handler._has_agentic_completion_hook(logging_obj) is True
# Unresolvable string (returns None) is skipped, no false positive.
monkeypatch.setattr(litellm, "callbacks", ["unknown_callback"])
monkeypatch.setattr(
"litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class",
lambda name: None,
)
assert handler._has_agentic_completion_hook(logging_obj) is False
def test_fingerprint_agentic_tools_is_deterministic():
handler = BaseLLMHTTPHandler()
tools_a = {"tool_calls": [{"id": "1", "input": {"q": "abc"}, "name": "web_search"}]}
tools_b = {"tool_calls": [{"name": "web_search", "input": {"q": "abc"}, "id": "1"}]}
assert handler._fingerprint_agentic_tools(tools_a) == handler._fingerprint_agentic_tools(tools_b)
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_extra_headers():
"""
Test that async_anthropic_messages_handler correctly extracts and merges
extra_headers from kwargs with proper priority.
"""
handler = BaseLLMHTTPHandler()
# Mock the config
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "test-key"}, "https://api.anthropic.com")
)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-3-opus-20240229", "messages": []}
)
# Mock the client
mock_client = AsyncMock()
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello!"}],
"model": "claude-3-opus-20240229",
"stop_reason": "end_turn",
}
mock_client.post = AsyncMock(return_value=mock_response)
# Mock logging object
mock_logging_obj = Mock()
mock_logging_obj.update_environment_variables = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
# Test case 1: Only extra_headers in kwargs
kwargs = {
"extra_headers": {
"X-Custom-Header": "from-kwargs",
"X-Auth-Token": "token123",
}
}
with patch(
"litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers"
) as mock_provider_headers:
mock_provider_headers.return_value = None
# Capture what headers are passed to validate_anthropic_messages_environment
captured_headers = {}
def capture_validate(*args, **kwargs):
captured_headers.update(kwargs.get("headers", {}))
return ({"x-api-key": "test-key"}, "https://api.anthropic.com")
mock_config.avalidate_anthropic_messages_environment = AsyncMock(side_effect=capture_validate)
try:
await handler.async_anthropic_messages_handler(
model="claude-3-opus-20240229",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
kwargs=kwargs,
)
except Exception:
pass # We're testing header extraction, not the full flow
# Verify extra_headers were extracted and merged
assert "X-Custom-Header" in captured_headers
assert captured_headers["X-Custom-Header"] == "from-kwargs"
assert "X-Auth-Token" in captured_headers
assert captured_headers["X-Auth-Token"] == "token123"
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_streaming_forwards_provider_response_headers():
"""
Regression test for LIT-3724 (issue 2): streaming /v1/messages responses
dropped the upstream provider's HTTP response headers, so Bedrock's
x-amzn-requestid / x-amzn-trace-id never reached clients even with
`return_response_headers: true`. The returned stream object must carry
them in `_hidden_params["additional_headers"]` (llm_provider-* prefixed),
which the proxy merges into the client-facing response headers.
"""
from collections.abc import AsyncIterator as ABCAsyncIterator
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
handler = BaseLLMHTTPHandler()
sse_body = (
b'event: message_start\ndata: {"type": "message_start"}\n\n'
b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
)
upstream_response = httpx.Response(
200,
headers={
"x-amzn-requestid": "amzn-req-123",
"x-amzn-trace-id": "Root=1-abc-def",
},
content=sse_body,
request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
)
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=upstream_response)
mock_logging_obj = Mock()
mock_logging_obj.model_call_details = {}
result = await handler.async_anthropic_messages_handler(
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=AnthropicMessagesConfig(),
anthropic_messages_optional_request_params={"max_tokens": 32},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
api_key="sk-test",
stream=True,
kwargs={},
)
assert isinstance(result, ABCAsyncIterator)
additional_headers = result._hidden_params["additional_headers"]
assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123"
assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def"
assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers)
collected = b"".join([chunk async for chunk in result])
assert b"message_start" in collected
assert b"message_stop" in collected
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_agentic_streaming_forwards_provider_response_headers():
"""
Companion to the test above for the agentic branch: when a callback
overrides async_should_run_agentic_loop, the handler wraps
AgenticAnthropicStreamingIterator in AnthropicMessagesStreamingResponse.
That wrapping must still expose the provider headers and delegate
iteration through the two-phase agentic iterator unchanged.
"""
from collections.abc import AsyncIterator as ABCAsyncIterator
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
AgenticAnthropicStreamingIterator,
)
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
class NoOpAgenticCallback(CustomLogger):
async def async_should_run_agentic_loop(
self,
response,
model,
messages,
tools,
stream,
custom_llm_provider,
kwargs,
):
return False, {}
handler = BaseLLMHTTPHandler()
sse_body = (
b'event: message_start\ndata: {"type": "message_start"}\n\n'
b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
)
upstream_response = httpx.Response(
200,
headers={"x-amzn-requestid": "amzn-req-456"},
content=sse_body,
request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
)
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=upstream_response)
mock_logging_obj = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.dynamic_success_callbacks = [NoOpAgenticCallback()]
result = await handler.async_anthropic_messages_handler(
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=AnthropicMessagesConfig(),
anthropic_messages_optional_request_params={"max_tokens": 32},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
api_key="sk-test",
stream=True,
kwargs={},
)
assert isinstance(result, ABCAsyncIterator)
assert isinstance(result.completion_stream, AgenticAnthropicStreamingIterator)
assert result._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "amzn-req-456"
collected = b"".join([chunk async for chunk in result])
assert b"message_start" in collected
assert b"message_stop" in collected
@pytest.mark.asyncio
async def test_anthropic_messages_streaming_response_aclose_closes_upstream_stream():
"""
Regression test: the proxy's streaming cleanup calls aclose on the
handler's return value (see _finalize_streaming_generator_cleanup's
hasattr(response, "aclose") check). The wrapper must forward aclose to
the upstream stream so provider connections are released on client
disconnect instead of lingering until garbage collection.
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
)
class UpstreamTracker:
def __init__(self):
self.closed = False
tracker = UpstreamTracker()
async def upstream():
try:
yield b'data: {"type": "message_start"}\n\n'
yield b'data: {"type": "message_stop"}\n\n'
finally:
tracker.closed = True
stream = AnthropicMessagesStreamingResponse(
completion_stream=upstream(),
hidden_params={"additional_headers": {}},
)
first_chunk = await stream.__anext__()
assert b"message_start" in first_chunk
assert tracker.closed is False
await stream.aclose()
assert tracker.closed is True
@pytest.mark.asyncio
async def test_anthropic_messages_streaming_response_aclose_closes_agentic_upstream_stream():
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
AgenticAnthropicStreamingIterator,
)
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
)
class UpstreamTracker:
def __init__(self):
self.closed = False
tracker = UpstreamTracker()
async def upstream():
try:
yield b'data: {"type": "message_start"}\n\n'
yield b'data: {"type": "message_stop"}\n\n'
finally:
tracker.closed = True
agentic_iterator = AgenticAnthropicStreamingIterator(
completion_stream=upstream(),
http_handler=Mock(),
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=Mock(),
anthropic_messages_optional_request_params={},
logging_obj=Mock(),
custom_llm_provider="anthropic",
kwargs={},
)
stream = AnthropicMessagesStreamingResponse(
completion_stream=agentic_iterator,
hidden_params={"additional_headers": {}},
)
first_chunk = await stream.__anext__()
assert b"message_start" in first_chunk
assert tracker.closed is False
await stream.aclose()
assert tracker.closed is True
class _ProbedBedrockMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
def __init__(self, probe: EventLoopProbe) -> None:
super().__init__()
self._probe = probe
def get_credentials(
self,
**kwargs: object, # kwargs-ok: mirrors the base resolver's keyword contract, which the probe ignores
) -> RefreshableCredentials:
return self._probe.credentials()
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_signs_bedrock_off_the_event_loop(monkeypatch):
"""Regression for issue #40165: /v1/messages on Bedrock signed on the loop, so botocore's blocking
credential refresh inside SigV4 stalled every other request on the worker."""
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
probe = EventLoopProbe()
handler = BaseLLMHTTPHandler()
upstream_response = httpx.Response(
200,
json={
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "hi"}],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
request=httpx.Request("POST", "https://bedrock-runtime.us-west-2.amazonaws.com/"),
)
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=upstream_response)
mock_logging_obj = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.dynamic_success_callbacks = None
release = asyncio.create_task(probe.release_refresh_from_the_loop())
await handler.async_anthropic_messages_handler(
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_provider_config=_ProbedBedrockMessagesConfig(probe),
anthropic_messages_optional_request_params={"max_tokens": 16},
custom_llm_provider="bedrock",
litellm_params=GenericLiteLLMParams(aws_region_name="us-west-2"),
logging_obj=mock_logging_obj,
client=mock_client,
stream=False,
kwargs={},
)
await release
sent_headers = mock_client.post.call_args.kwargs["headers"]
assert sent_headers["Authorization"].startswith("AWS4-HMAC-SHA256")
assert probe.served_during_refresh is True
_BEDROCK_MESSAGE: Final = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
}
_BEDROCK_RESPONSE: Final = {
"id": "resp_1",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "openai.gpt-5.6-sol",
"output": [],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
}
_EMPTY_S3_LISTING: Final = (
b'<?xml version="1.0" encoding="UTF-8"?>'
b'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
b"<Name>batch-inputs</Name><KeyCount>0</KeyCount><IsTruncated>false</IsTruncated></ListBucketResult>"
)
_BATCH_LINE: Final = json.dumps(
{
"custom_id": "1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": "us.anthropic.claude-sonnet-5-5", "messages": [{"role": "user", "content": "hi"}]},
}
).encode()
_BATCH_FILE_ID: Final = "s3://batch-inputs/litellm-bedrock-files/batch.jsonl"
_BEDROCK_ENTRY_POINTS: Final[tuple[tuple[str, Callable[[], Awaitable[object]], httpx.Response], ...]] = (
(
"anthropic_messages",
lambda: litellm.anthropic.messages.acreate(
model="bedrock/invoke/us.anthropic.claude-sonnet-5-5",
messages=[{"role": "user", "content": "hi"}],
max_tokens=16,
),
httpx.Response(200, json=_BEDROCK_MESSAGE),
),
(
"responses",
lambda: litellm.aresponses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
httpx.Response(200, json=_BEDROCK_RESPONSE),
),
(
"compact_responses",
lambda: litellm.acompact_responses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
httpx.Response(200, json=_BEDROCK_RESPONSE),
),
(
"search",
lambda: litellm.asearch(
query="hi",
search_provider="agentcore",
api_base="https://gw.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp",
),
httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"content": []}}),
),
(
"create_file",
lambda: litellm.acreate_file(
file=("batch.jsonl", _BATCH_LINE, "application/jsonl"),
purpose="batch",
custom_llm_provider="bedrock",
model="bedrock/us.anthropic.claude-sonnet-5-5",
),
httpx.Response(200),
),
(
"retrieve_file",
lambda: litellm.afile_retrieve(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
httpx.Response(200, content=_BATCH_LINE),
),
(
"delete_file",
lambda: litellm.afile_delete(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
httpx.Response(204),
),
(
"list_files",
lambda: litellm.afile_list(custom_llm_provider="bedrock"),
httpx.Response(200, content=_EMPTY_S3_LISTING),
),
(
"file_content",
lambda: litellm.afile_content(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
httpx.Response(200, content=_BATCH_LINE),
),
(
"create_batch",
lambda: litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=_BATCH_FILE_ID,
custom_llm_provider="bedrock",
model="bedrock/us.anthropic.claude-sonnet-5-5",
aws_batch_role_arn="arn:aws:iam::123456789012:role/batch",
s3_output_bucket_name="batch-outputs",
),
httpx.Response(200, json={"jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/batch"}),
),
(
"vector_store_search",
lambda: litellm.vector_stores.asearch(vector_store_id="KBID12345", query="hi", custom_llm_provider="bedrock"),
httpx.Response(200, json={"retrievalResults": []}),
),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("call", "upstream_answer"),
[pytest.param(call, answer, id=name) for name, call, answer in _BEDROCK_ENTRY_POINTS],
)
async def test_bedrock_secret_lookups_leave_the_event_loop_free(
call: Callable[[], Awaitable[object]],
upstream_answer: httpx.Response,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTTESTTESTTEST")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
monkeypatch.setenv("AWS_S3_BUCKET_NAME", "batch-inputs")
monkeypatch.setenv("AWS_S3_OUTPUT_BUCKET_NAME", "batch-outputs")
vault: Final = host_in_loop_probing_vault(
monkeypatch,
"AWS_REGION_NAME",
"AWS_BEDROCK_RUNTIME_ENDPOINT",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_S3_ENCRYPTION_KEY_ID",
"AWS_S3_BUCKET_OWNER",
"AGENTCORE_MCP_PROTOCOL_VERSION",
"AGENTCORE_SEARCH_TOOL_NAME",
"AGENTCORE_GATEWAY_TOKEN",
)
upstream: Final = respx_mock.route().mock(return_value=upstream_answer)
_ = await call()
assert vault.read.is_set()
assert not vault.read_on_loop.is_set()
assert ".us-west-2." in upstream.calls.last.request.url.host
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_passes_litellm_metadata():
"""Ensure litellm_metadata from kwargs is forwarded via update_from_kwargs.
Routes like /messages store model_info under kwargs['litellm_metadata'].
The handler must forward this so that use_custom_pricing_for_model can
detect custom pricing. Regression test for #23185.
"""
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "test-key"}, "https://api.anthropic.com")
)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-sonnet-4-20250514", "messages": []}
)
mock_client = AsyncMock()
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello!"}],
"model": "claude-sonnet-4-20250514",
"stop_reason": "end_turn",
}
mock_client.post = AsyncMock(return_value=mock_response)
mock_logging_obj = Mock()
mock_logging_obj.update_from_kwargs = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
custom_model_info = {
"id": "claude-sonnet-4-custom-pricing",
"input_cost_per_token": 0.0003,
"output_cost_per_token": 0.0015,
}
kwargs = {
"litellm_metadata": {
"model_info": custom_model_info,
"deployment": "anthropic/claude-sonnet-4-20250514",
},
}
try:
await handler.async_anthropic_messages_handler(
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
kwargs=kwargs,
)
except Exception:
pass
mock_logging_obj.update_from_kwargs.assert_called_once()
call_kwargs = mock_logging_obj.update_from_kwargs.call_args
kwargs_arg = (
call_kwargs.kwargs.get("kwargs", call_kwargs[1].get("kwargs", {}))
if call_kwargs.kwargs
else call_kwargs[1].get("kwargs", {})
)
assert "litellm_metadata" in kwargs_arg
assert kwargs_arg["litellm_metadata"]["model_info"] == custom_model_info
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_forwards_router_model_info():
"""Ensure router deployment model_info is forwarded into litellm_params.
The Router stamps kwargs['model_info'] on every deployment dispatch via
_update_kwargs_with_deployment. Downstream cooldown / success callbacks
(router.deployment_callback_on_failure, deployment_callback_on_success)
look up the deployment id via kwargs['litellm_params']['model_info']['id'].
If async_anthropic_messages_handler builds its own litellm_params dict
without forwarding model_info, the id is missing and cooldown is silently
skipped for /v1/messages requests under the Router.
"""
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "test-key"}, "https://api.anthropic.com")
)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-sonnet-4-20250514", "messages": []}
)
mock_client = AsyncMock()
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello!"}],
"model": "claude-sonnet-4-20250514",
"stop_reason": "end_turn",
}
mock_client.post = AsyncMock(return_value=mock_response)
mock_logging_obj = Mock()
mock_logging_obj.update_from_kwargs = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
deployment_model_info = {
"id": "deployment-123",
"db_model": False,
}
try:
await handler.async_anthropic_messages_handler(
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
kwargs={"model_info": deployment_model_info},
)
except Exception:
pass
mock_logging_obj.update_from_kwargs.assert_called_once()
call_kwargs = mock_logging_obj.update_from_kwargs.call_args
litellm_params_arg = (
call_kwargs.kwargs.get("litellm_params", call_kwargs[1].get("litellm_params", {}))
if call_kwargs.kwargs
else call_kwargs[1].get("litellm_params", {})
)
assert litellm_params_arg.get("model_info") == deployment_model_info
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_header_priority():
"""
Test that async_anthropic_messages_handler respects header priority:
forwarded < extra_headers < provider_specific
"""
handler = BaseLLMHTTPHandler()
# Mock the config
mock_config = Mock()
mock_client = AsyncMock()
mock_logging_obj = Mock()
mock_logging_obj.update_environment_variables = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
# Test with all three header sources
kwargs = {
"headers": {"X-Priority": "forwarded", "X-Forwarded-Only": "keep"},
"extra_headers": {"X-Priority": "extra", "X-Extra-Only": "also-keep"},
}
with patch(
"litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers"
) as mock_provider_headers:
mock_provider_headers.return_value = {
"X-Priority": "provider",
"X-Provider-Only": "keep-this-too",
}
captured_headers = {}
def capture_validate(*args, **kwargs):
captured_headers.update(kwargs.get("headers", {}))
return ({"x-api-key": "test-key"}, "https://api.anthropic.com")
mock_config.avalidate_anthropic_messages_environment = AsyncMock(side_effect=capture_validate)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-3-opus-20240229", "messages": []}
)
try:
await handler.async_anthropic_messages_handler(
model="claude-3-opus-20240229",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=mock_logging_obj,
client=mock_client,
kwargs=kwargs,
)
except Exception:
pass
# Verify priority: provider_specific should win
assert captured_headers["X-Priority"] == "provider"
# Verify all unique headers from different sources are present
assert captured_headers["X-Forwarded-Only"] == "keep"
assert captured_headers["X-Extra-Only"] == "also-keep"
assert captured_headers["X-Provider-Only"] == "keep-this-too"
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_drops_top_level_and_nested_params():
"""
Regression for LIT-3988 / GitHub #25931: on the /v1/messages path,
additional_drop_params must strip plain top-level keys (e.g. `thinking`,
`context_management`) before the provider transform runs, not only nested
dotted paths. Bedrock rejects these fields, so leaving them in produces a 400.
"""
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "test-key"}, "https://api.anthropic.com")
)
captured = {}
def capture_transform(*args, **kwargs):
captured["optional_params"] = kwargs["anthropic_messages_optional_request_params"]
return {"model": "claude-opus-4-7", "messages": []}
mock_config.transform_anthropic_messages_request = capture_transform
mock_client = AsyncMock()
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello!"}],
"model": "claude-opus-4-7",
"stop_reason": "end_turn",
}
mock_client.post = AsyncMock(return_value=mock_response)
mock_logging_obj = Mock()
mock_logging_obj.update_from_kwargs = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
optional_params = {
"max_tokens": 1024,
"thinking": {"type": "enabled", "budget_tokens": 2048},
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
"metadata": {"user_id": "u1", "drop_me": "x"},
}
with patch(
"litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers"
) as mock_provider_headers:
mock_provider_headers.return_value = None
try:
await handler.async_anthropic_messages_handler(
model="claude-opus-4-7",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params=optional_params,
custom_llm_provider="bedrock",
litellm_params=GenericLiteLLMParams(
additional_drop_params=[
"thinking",
"context_management",
"metadata.drop_me",
]
),
logging_obj=mock_logging_obj,
client=mock_client,
)
except Exception:
pass # drop runs before the mocked sign_request; the capture is what we assert on
transformed = captured["optional_params"]
assert "thinking" not in transformed
assert "context_management" not in transformed
assert transformed["max_tokens"] == 1024
assert transformed["metadata"] == {"user_id": "u1"}
def test_google_genai_streaming_hidden_params_model_info_and_router_fallback():
logging_obj = Mock()
logging_obj.get_router_model_id = Mock(return_value="router-model-id")
from_model_info = _google_genai_streaming_hidden_params(
api_base="https://generativelanguage.googleapis.com/v1beta",
litellm_params=GenericLiteLLMParams(model_info={"id": "info-id"}),
logging_obj=logging_obj,
response_headers=httpx.Headers({"x-ratelimit-remaining": "10"}),
)
assert from_model_info["model_id"] == "info-id"
assert from_model_info["api_base"] == "https://generativelanguage.googleapis.com/v1beta"
assert isinstance(from_model_info["additional_headers"], dict)
from_router = _google_genai_streaming_hidden_params(
api_base="https://x",
litellm_params=GenericLiteLLMParams(),
logging_obj=logging_obj,
response_headers=httpx.Headers({}),
)
assert from_router["model_id"] == "router-model-id"
def _build_delete_response_mock(captured: dict):
"""Returns a fake httpx delete that records its kwargs."""
def _response() -> httpx.Response:
return httpx.Response(
status_code=200,
headers={"content-type": "application/json"},
content=b'{"id": "resp_x", "object": "response", "deleted": true}',
request=httpx.Request(method="DELETE", url="https://test.openai.azure.com"),
)
async def fake_async_delete(*args, **kwargs):
captured.update(kwargs)
return _response()
def fake_sync_delete(*args, **kwargs):
captured.update(kwargs)
return _response()
return fake_async_delete, fake_sync_delete
def test_async_delete_responses_omits_body_for_azure():
"""Azure responses DELETE rejects requests with any body. Verify the handler
does not pass `json=` to httpx when the transformer returns an empty dict."""
captured: dict = {}
fake_async_delete, _ = _build_delete_response_mock(captured)
async def run():
with patch.object(AsyncHTTPHandler, "delete", new=fake_async_delete):
await litellm.adelete_responses(
response_id="resp_xyz",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
)
asyncio.run(run())
assert "json" not in captured
assert "data" not in captured
assert captured["url"].endswith("/openai/responses/resp_xyz?api-version=2025-03-01-preview")
def test_sync_delete_responses_omits_body_for_azure():
captured: dict = {}
_, fake_sync_delete = _build_delete_response_mock(captured)
with patch.object(HTTPHandler, "delete", new=fake_sync_delete):
litellm.delete_responses(
response_id="resp_xyz",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
)
assert "json" not in captured
assert "data" not in captured
assert captured["url"].endswith("/openai/responses/resp_xyz?api-version=2025-03-01-preview")
def _content_type(headers: dict) -> str:
for key, value in headers.items():
if key.lower() == "content-type":
return value
return ""
def test_async_delete_responses_sets_json_content_type():
"""OpenAI rejects a responses DELETE with no Content-Type by treating it as
application/octet-stream. The handler must declare application/json."""
captured: dict = {}
fake_async_delete, _ = _build_delete_response_mock(captured)
async def run():
with patch.object(AsyncHTTPHandler, "delete", new=fake_async_delete):
await litellm.adelete_responses(
response_id="resp_xyz",
custom_llm_provider="openai",
api_key="test-key",
)
asyncio.run(run())
assert _content_type(captured["headers"]) == "application/json"
def test_sync_delete_responses_sets_json_content_type():
captured: dict = {}
_, fake_sync_delete = _build_delete_response_mock(captured)
with patch.object(HTTPHandler, "delete", new=fake_sync_delete):
litellm.delete_responses(
response_id="resp_xyz",
custom_llm_provider="openai",
api_key="test-key",
)
assert _content_type(captured["headers"]) == "application/json"
# ---------------------------------------------------------------------------
# Parity tests: request-body is serialized once and reused for the wire.
# (_async_post_anthropic_messages_with_http_error_retry)
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"litellm_params_kwargs, stream, global_timeout, expected",
[
({"timeout": 12.0}, False, None, 12.0),
({"request_timeout": 30.0}, False, None, 30.0),
({}, False, 1500.0, 1500.0),
({"timeout": 5.0, "stream_timeout": 50.0}, True, None, 50.0),
({"timeout": 5.0, "stream_timeout": 50.0}, False, None, 5.0),
({"timeout": 5.0, "request_timeout": 30.0}, False, None, 5.0),
({}, False, None, None),
({}, True, None, None),
],
)
def test_resolve_anthropic_messages_timeout(monkeypatch, litellm_params_kwargs, stream, global_timeout, expected):
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
if global_timeout is None:
monkeypatch.setattr(
"litellm.request_timeout",
float(DEFAULT_REQUEST_TIMEOUT_SECONDS),
raising=False,
)
monkeypatch.setattr(
"litellm.request_timeout_explicitly_set",
False,
raising=False,
)
else:
monkeypatch.setattr("litellm.request_timeout", global_timeout, raising=False)
monkeypatch.setattr("litellm.request_timeout_explicitly_set", True, raising=False)
resolved = BaseLLMHTTPHandler._resolve_anthropic_messages_timeout(
litellm_params=GenericLiteLLMParams(**litellm_params_kwargs),
stream=stream,
custom_llm_provider="anthropic",
)
assert resolved == expected
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeypatch):
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "request_timeout", float(DEFAULT_REQUEST_TIMEOUT_SECONDS))
monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False)
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "k"}, "https://api.anthropic.com")
)
mock_config.should_filter_anthropic_beta_headers = Mock(return_value=False)
mock_config.transform_anthropic_messages_request = Mock(return_value={"model": "claude", "messages": []})
mock_config.get_complete_url = Mock(return_value="https://api.anthropic.com/v1/messages")
mock_config.sign_request = Mock(return_value=({"x-api-key": "k"}, None))
mock_config.max_retry_on_anthropic_messages_http_error = 1
expected_response = {"id": "msg_1", "content": []}
mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response)
ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=ok_response)
logging_obj = Mock()
logging_obj.model_call_details = {}
logging_obj.dynamic_success_callbacks = []
result = await handler.async_anthropic_messages_handler(
model="claude",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(request_timeout=0.3),
logging_obj=logging_obj,
client=mock_client,
kwargs={},
)
assert result is expected_response
assert mock_client.post.await_args.kwargs["timeout"] == 0.3
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_forwards_stream_timeout(monkeypatch):
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "request_timeout", float(DEFAULT_REQUEST_TIMEOUT_SECONDS))
monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False)
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "k"}, "https://api.anthropic.com")
)
mock_config.should_filter_anthropic_beta_headers = Mock(return_value=False)
mock_config.transform_anthropic_messages_request = Mock(return_value={"model": "claude", "messages": []})
mock_config.get_complete_url = Mock(return_value="https://api.anthropic.com/v1/messages")
mock_config.sign_request = Mock(return_value=({"x-api-key": "k"}, None))
mock_config.max_retry_on_anthropic_messages_http_error = 1
mock_config.get_async_streaming_response_iterator = Mock(return_value=Mock())
ok_response = Mock()
ok_response.raise_for_status = Mock(return_value=None)
ok_response.headers = httpx.Headers({})
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=ok_response)
logging_obj = Mock()
logging_obj.model_call_details = {}
logging_obj.dynamic_success_callbacks = []
await handler.async_anthropic_messages_handler(
model="claude",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(timeout=9.0, stream_timeout=0.7),
logging_obj=logging_obj,
client=mock_client,
stream=True,
kwargs={},
)
assert mock_client.post.await_args.kwargs["stream"] is True
assert mock_client.post.await_args.kwargs["timeout"] == 0.7
@pytest.mark.asyncio
async def test_anthropic_post_uses_prebuilt_body_without_redumping():
"""When the caller passes a pre-serialized (unsigned) body, attempt 0 must
send exactly those bytes -- no second json.dumps of request_body."""
import json as _json
handler = BaseLLMHTTPHandler()
request_body = {"model": "claude", "messages": [{"role": "user", "content": "hi"}]}
prebuilt = _json.dumps(request_body)
ok_resp = Mock()
ok_resp.raise_for_status = Mock(return_value=None)
http_client = Mock()
http_client.post = AsyncMock(return_value=ok_resp)
provider_config = Mock()
provider_config.max_retry_on_anthropic_messages_http_error = 2
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
out = await handler._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=http_client,
request_url="http://x/v1/messages",
headers={},
signed_json_body=prebuilt,
request_body=request_body,
stream=False,
logging_obj=logging_obj,
provider_config=provider_config,
litellm_params=GenericLiteLLMParams(),
api_key="k",
model="claude",
)
assert out is ok_resp
http_client.post.assert_awaited_once()
sent = http_client.post.await_args.kwargs["data"]
# Byte-identical to the legacy wire serialization, and the SAME object the
# caller already used for the pre-call log (no re-serialization).
assert sent == prebuilt
assert sent is prebuilt
@pytest.mark.asyncio
async def test_anthropic_post_falls_back_to_json_dumps_when_unsigned_none():
"""signed_json_body=None keeps the exact legacy behavior."""
import json as _json
handler = BaseLLMHTTPHandler()
request_body = {"model": "claude", "messages": [{"role": "user", "content": "yo"}]}
ok_resp = Mock()
ok_resp.raise_for_status = Mock(return_value=None)
http_client = Mock()
http_client.post = AsyncMock(return_value=ok_resp)
provider_config = Mock()
provider_config.max_retry_on_anthropic_messages_http_error = 1
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
await handler._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=http_client,
request_url="http://x/v1/messages",
headers={},
signed_json_body=None,
request_body=request_body,
stream=False,
logging_obj=logging_obj,
provider_config=provider_config,
litellm_params=GenericLiteLLMParams(),
api_key="k",
model="claude",
)
sent = http_client.post.await_args.kwargs["data"]
assert sent == _json.dumps(request_body)
@pytest.mark.asyncio
async def test_anthropic_post_retry_reserializes_mutated_body():
"""On a retryable HTTP error the body is mutated + re-signed; the prebuilt
body must NOT be reused -- attempt 1 sends the freshly serialized body."""
import json as _json
handler = BaseLLMHTTPHandler()
request_body = {"model": "claude", "messages": [{"role": "user", "content": "a"}]}
prebuilt = _json.dumps(request_body)
err_resp = Mock()
http_error = httpx.HTTPStatusError("bad", request=Mock(), response=Mock(status_code=400))
err_resp.raise_for_status = Mock(side_effect=http_error)
ok_resp = Mock()
ok_resp.raise_for_status = Mock(return_value=None)
http_client = Mock()
http_client.post = AsyncMock(side_effect=[err_resp, ok_resp])
def _mutate(e, request_data):
request_data["messages"][0]["content"] = "MUTATED"
provider_config = Mock()
provider_config.max_retry_on_anthropic_messages_http_error = 2
provider_config.should_retry_anthropic_messages_on_http_error = Mock(return_value=True)
provider_config.transform_anthropic_messages_request_on_http_error = _mutate
# Re-sign returns no signed body (native anthropic path) -> must re-dump.
provider_config.sign_request = Mock(return_value=({}, None))
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
await handler._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=http_client,
request_url="http://x/v1/messages",
headers={},
signed_json_body=prebuilt,
request_body=request_body,
stream=False,
logging_obj=logging_obj,
provider_config=provider_config,
litellm_params=GenericLiteLLMParams(),
api_key="k",
model="claude",
)
assert http_client.post.await_count == 2
first_sent = http_client.post.await_args_list[0].kwargs["data"]
second_sent = http_client.post.await_args_list[1].kwargs["data"]
assert first_sent == prebuilt # attempt 0 used prebuilt
assert second_sent == _json.dumps(request_body) # attempt 1 re-serialized
assert "MUTATED" in second_sent # ... the mutated body
_BOUND_THINKING_ERROR: Final = (
'{"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0: '
'Invalid `signature` in `thinking` block. The block is bound to a different conversation."}}'
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("prefix_mismatch_behavior", "expected_posts"),
(("error", 1), ("drop_block", 2)),
)
async def test_anthropic_post_skips_the_thinking_strip_retry_when_block_binding_errors(
prefix_mismatch_behavior: str, expected_posts: int
) -> None:
"""A caller that pinned ``prefix_mismatch_behavior: error`` asked for the 400,
so the handler must surface it instead of stripping the bound blocks and retrying."""
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
handler = BaseLLMHTTPHandler()
thinking_block = {"type": "thinking", "thinking": "", "signature": "sig"}
request_body = {
"model": "claude-opus-5-5",
"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": [thinking_block, {"type": "text", "text": "hello"}]},
],
"thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": prefix_mismatch_behavior}},
}
request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")
http_client = Mock()
http_client.post = AsyncMock(
side_effect=[
httpx.Response(400, request=request, text=_BOUND_THINKING_ERROR),
httpx.Response(200, request=request, json={"type": "message", "content": []}),
]
)
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
async def _post() -> httpx.Response:
return await handler._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=http_client,
request_url="https://api.anthropic.com/v1/messages",
headers={},
signed_json_body=None,
request_body=request_body,
stream=False,
logging_obj=logging_obj,
provider_config=AnthropicMessagesConfig(),
litellm_params=GenericLiteLLMParams(),
api_key="k",
model="claude-opus-5-5",
)
if expected_posts == 1:
with pytest.raises(BaseLLMException) as raised:
await _post()
assert raised.value.status_code == 400
assert "bound to a different conversation" in raised.value.message
assert request_body["messages"][1]["content"][0] == thinking_block
else:
assert (await _post()).status_code == 200
assert http_client.post.await_count == expected_posts
def test_base_responses_config_sign_request_is_noop_by_default():
"""Default responses sign_request must be a no-op: unchanged headers, no signed body.
Guards the 15 existing responses providers from accidental signing when the
handler starts calling sign_request.
"""
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
cfg = OpenAIResponsesAPIConfig()
headers = {"Authorization": "Bearer sk-existing"}
out_headers, signed_body = cfg.sign_request(
headers=headers,
optional_params={},
request_data={"input": "hi"},
api_base="https://api.openai.com/v1/responses",
)
assert out_headers == {"Authorization": "Bearer sk-existing"}
assert signed_body is None
def _make_responses_handler_call(signed_body):
"""Drive BaseLLMHTTPHandler.response_api_handler with a fully mocked provider
config + sync client, returning the kwargs the client.post was called with.
signed_body=None simulates a no-op (non-signing) provider; bytes simulates a
signing provider (e.g. Bedrock Mantle).
"""
from unittest.mock import MagicMock
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.router import GenericLiteLLMParams
provider_config = MagicMock()
provider_config.validate_environment.return_value = {}
provider_config.get_complete_url.return_value = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses"
provider_config.transform_responses_api_request.return_value = {"input": "hi"}
provider_config.should_fake_stream.return_value = False
provider_config.sign_request.return_value = ({"X-Signed": "1"}, signed_body)
mock_client = MagicMock(spec=HTTPHandler)
mock_client.post.return_value = MagicMock()
handler = BaseLLMHTTPHandler()
handler.response_api_handler(
model="openai.gpt-5.5",
input="hi",
responses_api_provider_config=provider_config,
response_api_optional_request_params={},
custom_llm_provider="bedrock_mantle",
litellm_params=GenericLiteLLMParams(aws_region_name="us-east-2"),
logging_obj=MagicMock(),
client=mock_client,
_is_async=False,
)
return mock_client.post.call_args.kwargs
def test_responses_handler_sends_json_when_not_signed():
"""No-op provider (signed_body is None) -> handler posts json=data, no data= bytes."""
kwargs = _make_responses_handler_call(signed_body=None)
assert kwargs.get("json") == {"input": "hi"}
assert "data" not in kwargs
def test_responses_handler_sends_signed_bytes_when_signed():
"""Signing provider -> handler posts the exact signed bytes via data=, not json=."""
kwargs = _make_responses_handler_call(signed_body=b'{"input": "hi"}')
assert kwargs.get("data") == b'{"input": "hi"}'
assert "json" not in kwargs
assert kwargs["headers"] == {"X-Signed": "1"}
def test_responses_handler_signs_after_fake_stream_prep_strips_stream():
"""Fake-stream signing-order invariant: the bytes SIGNED must equal the bytes SENT.
In the streaming + fake-stream path the handler first runs
_prepare_fake_stream_request, which pops "stream" out of the body, and only
then calls sign_request. If signing ran before that pop, the signed body
would still carry "stream" while the body sent over the wire would not,
producing a SigV4 payload-hash mismatch (401) for a real Mantle deployment.
We snapshot request_data at sign time and assert "stream" is already gone.
"""
from unittest.mock import MagicMock
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import GenericLiteLLMParams
provider_config = MagicMock()
provider_config.validate_environment.return_value = {}
provider_config.get_complete_url.return_value = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses"
provider_config.transform_responses_api_request.return_value = {
"input": "hi",
"stream": True,
}
provider_config.should_fake_stream.return_value = True
provider_config.transform_response_api_response.return_value = ResponsesAPIResponse(
id="resp_1",
created_at=0,
output=[],
status="completed",
model="openai.gpt-5.5",
)
captured = {}
def _capture_sign(**kwargs):
captured["request_data"] = dict(kwargs["request_data"])
return ({"X-Signed": "1"}, b'{"input": "hi"}')
provider_config.sign_request.side_effect = _capture_sign
mock_client = MagicMock(spec=HTTPHandler)
mock_client.post.return_value = MagicMock()
handler = BaseLLMHTTPHandler()
handler.response_api_handler(
model="openai.gpt-5.5",
input="hi",
responses_api_provider_config=provider_config,
response_api_optional_request_params={"stream": True},
custom_llm_provider="bedrock_mantle",
litellm_params=GenericLiteLLMParams(aws_region_name="us-east-2"),
logging_obj=MagicMock(),
client=mock_client,
_is_async=False,
fake_stream=True,
)
assert "stream" not in captured["request_data"]
assert "input" in captured["request_data"]
post_kwargs = mock_client.post.call_args.kwargs
assert post_kwargs.get("data") == b'{"input": "hi"}'
assert "json" not in post_kwargs
assert "stream" in post_kwargs
def _make_compact_handler_call(signed_body, is_async):
"""Drive (async_)compact_response_api_handler with a fully mocked provider config
+ client, returning the kwargs the client.post was called with.
signed_body=None simulates a no-op (non-signing) provider; bytes simulates a
signing provider (e.g. Bedrock Mantle SigV4 / bearer).
"""
from unittest.mock import MagicMock
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.router import GenericLiteLLMParams
compact_url = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses/compact"
provider_config = MagicMock()
provider_config.validate_environment.return_value = {}
provider_config.get_complete_url.return_value = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses"
provider_config.transform_compact_response_api_request.return_value = (
compact_url,
{"model": "openai.gpt-5.5", "input": "hi"},
)
provider_config.sign_request.return_value = ({"X-Signed": "1"}, signed_body)
provider_config.transform_compact_response_api_response.return_value = "ok"
spec = AsyncHTTPHandler if is_async else HTTPHandler
mock_client = MagicMock(spec=spec)
if is_async:
mock_client.post = AsyncMock(return_value=MagicMock())
else:
mock_client.post.return_value = MagicMock()
handler = BaseLLMHTTPHandler()
result = handler.compact_response_api_handler(
model="openai.gpt-5.5",
input="hi",
responses_api_provider_config=provider_config,
response_api_optional_request_params={},
custom_llm_provider="bedrock_mantle",
litellm_params=GenericLiteLLMParams(aws_region_name="us-east-2"),
logging_obj=MagicMock(),
client=mock_client,
_is_async=is_async,
)
if is_async:
asyncio.run(result)
return provider_config, mock_client.post.call_args.kwargs
def test_compact_handler_sends_json_when_not_signed():
"""No-op provider on compact (signed_body is None) -> posts json=data, no data= bytes."""
provider_config, kwargs = _make_compact_handler_call(signed_body=None, is_async=False)
provider_config.sign_request.assert_called_once()
assert kwargs.get("json") == {"model": "openai.gpt-5.5", "input": "hi"}
assert "data" not in kwargs
def test_compact_handler_sends_signed_bytes_when_signed():
"""Signing provider on compact -> posts the signed bytes via data=, not json=.
Regression for the adversarial-review finding that /responses/compact bypassed
the SigV4 signing hook, so IAM-only Mantle callers sent unsigned bodies.
"""
provider_config, kwargs = _make_compact_handler_call(
signed_body=b'{"model": "openai.gpt-5.5", "input": "hi"}', is_async=False
)
assert kwargs.get("data") == b'{"model": "openai.gpt-5.5", "input": "hi"}'
assert "json" not in kwargs
assert kwargs["headers"] == {"X-Signed": "1"}
# signing must use the compact endpoint as api_base, not the create URL
assert provider_config.sign_request.call_args.kwargs["api_base"].endswith("/openai/v1/responses/compact")
def test_async_compact_handler_sends_signed_bytes_when_signed():
"""Async compact must sign identically to sync (same omission in the async twin)."""
provider_config, kwargs = _make_compact_handler_call(
signed_body=b'{"model": "openai.gpt-5.5", "input": "hi"}', is_async=True
)
assert kwargs.get("data") == b'{"model": "openai.gpt-5.5", "input": "hi"}'
assert "json" not in kwargs
assert kwargs["headers"] == {"X-Signed": "1"}
def test_async_compact_handler_sends_json_when_not_signed():
"""Async no-op provider on compact -> posts json=data, no data= bytes."""
_provider_config, kwargs = _make_compact_handler_call(signed_body=None, is_async=True)
assert kwargs.get("json") == {"model": "openai.gpt-5.5", "input": "hi"}
assert "data" not in kwargs
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks():
"""
Regression: async_anthropic_messages_handler must inject api_key into the
kwargs dict forwarded to _call_agentic_completion_hooks.
Without this, follow-up calls made by agentic hooks (e.g. websearch
interception's second LLM call after executing searches) have no api_key
and fail with "x-api-key header is required".
"""
handler = BaseLLMHTTPHandler()
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"x-api-key": "sk-test"}, "https://api.anthropic.com")
)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-haiku", "messages": [], "max_tokens": 16}
)
mock_config.sign_request = Mock(return_value=({}, None))
fake_raw_response = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"content": [],
"stop_reason": "end_turn",
}
mock_config.transform_anthropic_messages_response = Mock(return_value=fake_raw_response)
mock_logging_obj = Mock()
mock_logging_obj.update_environment_variables = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
mock_logging_obj.dynamic_success_callbacks = None
captured_kwargs: dict = {}
sentinel_response = object()
async def fake_agentic_hooks(**call_kwargs):
captured_kwargs.update(call_kwargs)
return sentinel_response
mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
with (
patch.object(
handler,
"_async_post_anthropic_messages_with_http_error_retry",
new=AsyncMock(return_value=mock_httpx_response),
),
patch.object(handler, "_call_agentic_completion_hooks", side_effect=fake_agentic_hooks),
patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client"),
patch( # test-quality-ok: the proxy wiring under test is what this patches
"litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers",
return_value=None,
),
):
result = await handler.async_anthropic_messages_handler(
model="claude-haiku",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={"stream": False},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(api_key="sk-real-anthropic-key"),
logging_obj=mock_logging_obj,
api_key="sk-real-anthropic-key",
stream=False,
)
assert result is sentinel_response
assert "kwargs" in captured_kwargs, "_call_agentic_completion_hooks not called"
forwarded = captured_kwargs["kwargs"]
assert forwarded.get("api_key") == "sk-real-anthropic-key", (
"api_key must be injected into kwargs passed to _call_agentic_completion_hooks "
"so follow-up calls in agentic hooks (e.g. websearch) can authenticate"
)
_FOUNDRY_API_BASE: Final = "https://lit5418.services.ai.azure.com/anthropic"
_FOUNDRY_SSE_BODY: Final = (
b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_1", "type": "message", '
b'"role": "assistant", "model": "claude-fable-5-1", "content": [], "stop_reason": null, '
b'"usage": {"input_tokens": 1, "output_tokens": 0}}}\n\n'
b'event: content_block_start\ndata: {"type": "content_block_start", "index": 0, '
b'"content_block": {"type": "text", "text": ""}}\n\n'
b'event: content_block_delta\ndata: {"type": "content_block_delta", "index": 0, '
b'"delta": {"type": "text_delta", "text": "ready"}}\n\n'
b'event: content_block_stop\ndata: {"type": "content_block_stop", "index": 0}\n\n'
b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, '
b'"usage": {"output_tokens": 1}}\n\n'
b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
)
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_passes_deployment_api_base_to_agentic_hooks(stream, monkeypatch):
"""
Regression for LIT-5418: an azure_ai deployment carries its Foundry endpoint as
``api_base``, a named parameter that never lands in kwargs. The agentic hooks
(websearch interception's follow-up call after the search) must receive it on
both the non-streaming and the streaming path, or the follow-up fails with
"Missing Azure API Base" and the client gets the dangling tool_use back.
"""
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
monkeypatch.delenv("AZURE_API_BASE", raising=False)
class CapturingAgenticCallback(CustomLogger):
def __init__(self):
super().__init__()
self.hook_kwargs: dict | None = None
async def async_should_run_agentic_loop(self, response, model, messages, tools, stream, custom_llm_provider, kwargs):
self.hook_kwargs = dict(kwargs)
return False, {}
callback = CapturingAgenticCallback()
handler = BaseLLMHTTPHandler()
upstream_request = httpx.Request("POST", f"{_FOUNDRY_API_BASE}/v1/messages")
upstream_response = (
httpx.Response(200, content=_FOUNDRY_SSE_BODY, request=upstream_request)
if stream
else httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-fable-5-1",
"content": [{"type": "text", "text": "ready"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
request=upstream_request,
)
)
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=upstream_response)
mock_logging_obj = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.dynamic_success_callbacks = [callback]
result = await handler.async_anthropic_messages_handler(
model="claude-fable-5-1",
messages=[{"role": "user", "content": "Say ready"}],
anthropic_messages_provider_config=AzureAnthropicMessagesConfig(),
anthropic_messages_optional_request_params={"max_tokens": 32},
custom_llm_provider="azure_ai",
litellm_params=GenericLiteLLMParams(api_key="foundry-key", api_base=_FOUNDRY_API_BASE),
logging_obj=mock_logging_obj,
client=mock_client,
api_key="foundry-key",
api_base=_FOUNDRY_API_BASE,
stream=stream,
kwargs={},
)
if stream:
_ = [chunk async for chunk in result]
assert mock_client.post.call_args.kwargs["url"] == f"{_FOUNDRY_API_BASE}/v1/messages"
assert callback.hook_kwargs is not None, "agentic hook never ran"
assert callback.hook_kwargs.get("api_base") == _FOUNDRY_API_BASE
assert callback.hook_kwargs.get("api_key") == "foundry-key"
class _FakeWSExceptions:
class WebSocketException(Exception):
pass
class InvalidStatusCode(WebSocketException):
def __init__(self) -> None:
super().__init__("HTTP 403")
# websockets>=15 raises InvalidStatus (not InvalidStatusCode) for a rejected
# client handshake; both must be treated as deterministic.
class InvalidStatus(WebSocketException):
def __init__(self) -> None:
super().__init__("HTTP 401")
class _FakeWebsocketsModule:
"""Stand-in for the ``websockets`` module so the realtime backend-open retry
can be exercised without a real network handshake (dependency injection,
no monkeypatching)."""
def __init__(self, outcomes):
# outcomes: list where each item is either an Exception to raise or a
# sentinel object to return as the "connected" websocket.
self._outcomes = list(outcomes)
self.exceptions = _FakeWSExceptions
self.attempts = 0
self.open_timeouts: list = []
async def connect(self, *args, **kwargs):
self.attempts += 1
self.open_timeouts.append(kwargs.get("open_timeout"))
outcome = self._outcomes.pop(0)
if isinstance(outcome, Exception):
raise outcome
return outcome
@pytest.mark.asyncio
async def test_realtime_backend_open_retries_then_succeeds():
"""A hung/slow open handshake is retried; a later fresh attempt connects.
Regression for intermittent ``1011 timed out during opening handshake``:
the proxy used to surface a single slow upstream handshake to the caller as
a fatal 1011 with no retry.
"""
sentinel = object()
fake = _FakeWebsocketsModule([TimeoutError("timed out during opening handshake"), sentinel])
result = await BaseLLMHTTPHandler._open_realtime_backend_ws(
fake, "wss://backend.example/live", {"Authorization": "Bearer x"}, None
)
assert result is sentinel
assert fake.attempts == 2
# Each attempt must be bounded by a finite open_timeout (not the default/None).
assert all(t is not None and t > 0 for t in fake.open_timeouts)
@pytest.mark.asyncio
async def test_realtime_backend_open_raises_after_max_attempts():
"""When every attempt times out, the final error propagates (so the caller
still closes the client socket) rather than looping forever."""
fake = _FakeWebsocketsModule([TimeoutError("hang")] * 2)
with pytest.raises(TimeoutError):
await BaseLLMHTTPHandler._open_realtime_backend_ws(fake, "wss://backend.example/live", {}, None, max_attempts=2)
assert fake.attempts == 2
@pytest.mark.asyncio
@pytest.mark.parametrize(
"rejection",
[_FakeWSExceptions.InvalidStatusCode, _FakeWSExceptions.InvalidStatus],
)
async def test_realtime_backend_open_does_not_retry_auth_failure(rejection):
"""A deterministic handshake-status rejection (auth/4xx) must not be retried;
retrying cannot help and the upstream status must surface, not a 1011. Both
the websockets<15 (InvalidStatusCode) and >=15 (InvalidStatus) shapes apply."""
fake = _FakeWebsocketsModule([rejection()])
with pytest.raises(_FakeWSExceptions.WebSocketException):
await BaseLLMHTTPHandler._open_realtime_backend_ws(fake, "wss://backend.example/live", {}, None)
assert fake.attempts == 1
class _FakeClientWebSocket:
def __init__(self, send_error=None):
self.events = []
self._send_error = send_error
async def send_text(self, payload):
if self._send_error is not None:
raise self._send_error
self.events.append(("send_text", payload))
async def close(self, code=None, reason=None):
self.events.append(("close", (code, reason)))
async def _run_async_realtime_with_backend_failure(client_ws):
import websockets.exceptions # noqa: F401 # binds the submodule so async_realtime's except clause resolves, as in the proxy process
handler = BaseLLMHTTPHandler()
provider_config = Mock()
provider_config.get_complete_url.return_value = "wss://backend.example/live"
provider_config.validate_environment.return_value = {}
provider_config.open_backend = AsyncMock(return_value=None)
with patch.object(
handler,
"_open_realtime_backend_ws",
AsyncMock(side_effect=Exception("vertex token refresh exploded")),
):
await handler.async_realtime(
model="gemini-live-2.5-flash",
websocket=client_ws,
logging_obj=Mock(),
provider_config=provider_config,
headers={},
)
@pytest.mark.asyncio
async def test_async_realtime_generic_failure_sends_error_event_then_reasoned_close():
"""Regression for the realtime accept-then-silence hang: a generic backend
failure used to close the client socket without any error event, so callers
only saw a bare 1011. The client must receive an OpenAI-style error event
before the reasoned close."""
client_ws = _FakeClientWebSocket()
await _run_async_realtime_with_backend_failure(client_ws)
assert [name for name, _ in client_ws.events] == ["send_text", "close"]
error_event = json.loads(client_ws.events[0][1])
assert error_event["type"] == "error"
assert error_event["error"]["type"] == "server_error"
assert "vertex token refresh exploded" in error_event["error"]["message"]
assert client_ws.events[1][1] == (1011, "Internal server error: vertex token refresh exploded")
@pytest.mark.asyncio
async def test_async_realtime_error_event_send_failure_still_closes():
"""A client socket that already dropped must not turn the loud-failure path
into a new exception: the error-event send may fail, but the reasoned close
must still be attempted."""
client_ws = _FakeClientWebSocket(send_error=RuntimeError("client already disconnected"))
await _run_async_realtime_with_backend_failure(client_ws)
assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))]
class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
def get_supported_openai_params(self, model):
return []
def map_openai_params(self, non_default_params, optional_params, model, drop_params):
return optional_params
def validate_environment(
self,
headers,
model,
messages,
optional_params,
litellm_params,
api_key=None,
api_base=None,
):
return {**headers, "Authorization": "Bearer test-token"}
def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None):
return "https://transcription.example/recognize"
def transform_audio_transcription_request(self, model, audio_file, optional_params, litellm_params):
return AudioTranscriptionRequestData(data={"config": {"model": model}, "content": "YXVkaW8="})
def transform_audio_transcription_response(self, raw_response):
return TranscriptionResponse(text=raw_response.json()["text"])
def get_error_class(self, error_message, status_code, headers):
return BaseLLMException(message=error_message, status_code=status_code, headers=headers)
def _json_transcription_call_kwargs(provider_config):
return {
"model": "test-model",
"audio_file": b"raw-audio",
"optional_params": {},
"litellm_params": {},
"model_response": TranscriptionResponse(),
"timeout": 10.0,
"max_retries": 0,
"logging_obj": Mock(),
"api_key": None,
"api_base": None,
"custom_llm_provider": "custom",
"headers": {},
"provider_config": provider_config,
}
def _capture_json_transcription_request(captured):
def respond(request):
captured["content_type"] = request.headers.get("content-type")
captured["body"] = json.loads(request.content)
return httpx.Response(200, json={"text": "transcribed"})
return respond
def test_audio_transcriptions_sends_dict_data_as_json_body():
"""Regression: dict request data was passed to httpx's data= param, which
form-encodes it and silently ignores json=; JSON-body providers (e.g.
Google Speech-to-Text) need an application/json body."""
captured = {}
client = HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(_capture_json_transcription_request(captured)))
)
response = BaseLLMHTTPHandler().audio_transcriptions(
client=client,
atranscription=False,
**_json_transcription_call_kwargs(_JSONBodyAudioTranscriptionConfig()),
)
assert captured["content_type"] == "application/json"
assert captured["body"] == {"config": {"model": "test-model"}, "content": "YXVkaW8="}
assert response.text == "transcribed"
@pytest.mark.asyncio
async def test_async_audio_transcriptions_sends_dict_data_as_json_body():
captured = {}
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture_json_transcription_request(captured)))
response = await BaseLLMHTTPHandler().async_audio_transcriptions(
client=client,
**_json_transcription_call_kwargs(_JSONBodyAudioTranscriptionConfig()),
)
assert captured["content_type"] == "application/json"
assert captured["body"] == {"config": {"model": "test-model"}, "content": "YXVkaW8="}
assert response.text == "transcribed"
class _WordTimestampAudioTranscriptionConfig(_JSONBodyAudioTranscriptionConfig):
def transform_audio_transcription_response(self, raw_response):
payload = raw_response.json()
response = TranscriptionResponse(text=payload["text"])
response["words"] = payload["words"]
return response
def test_transform_audio_transcription_response_without_subtitle_opt_in_keeps_text_and_words():
words = [
{"word": "hello", "start": 0.0, "end": 0.5},
{"word": "world", "start": 0.5, "end": 1.0},
]
raw_response = httpx.Response(200, json={"text": "hello world", "words": words})
response = BaseLLMHTTPHandler()._transform_audio_transcription_response(
provider_config=_WordTimestampAudioTranscriptionConfig(),
model="test-model",
response=raw_response,
model_response=TranscriptionResponse(),
logging_obj=Mock(),
optional_params={"response_format": "srt"},
api_key=None,
)
assert response.text == "hello world"
assert response["words"] == words
class _SubtitleSynthesisAudioTranscriptionConfig(_JSONBodyAudioTranscriptionConfig):
@property
def supports_subtitle_synthesis(self) -> bool:
return True
def transform_audio_transcription_response(self, raw_response):
payload = raw_response.json()
response = TranscriptionResponse(text=payload["text"])
if "words" in payload:
response["words"] = payload["words"]
return response
def _transform_subtitle_response(payload):
return BaseLLMHTTPHandler()._transform_audio_transcription_response(
provider_config=_SubtitleSynthesisAudioTranscriptionConfig(),
model="test-model",
response=httpx.Response(200, json=payload),
model_response=TranscriptionResponse(),
logging_obj=Mock(),
optional_params={"response_format": "srt"},
api_key=None,
)
def test_subtitle_synthesis_fallback_without_timings_drops_words():
response = _transform_subtitle_response(
{"text": "hello world", "words": [{"word": "hello"}, {"word": "world"}]}
)
assert response.text == "hello world"
assert "words" not in response
def test_subtitle_synthesis_without_words_keeps_plain_text():
response = _transform_subtitle_response({"text": "hello world"})
assert response.text == "hello world"
assert "words" not in response
@pytest.mark.asyncio
async def test_async_retrieve_file_content_raises_on_http_error():
"""
LIT-4008 regression: a provider error response (e.g. Anthropic's 400
"File id must have `file_` prefix") must raise instead of being wrapped
as file content, which downstream batch cost tracking would parse as an
empty results file and bill $0.
"""
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.files.transformation import AnthropicFilesConfig
handler = BaseLLMHTTPHandler()
client = Mock(spec=AsyncHTTPHandler)
client.get = AsyncMock(
return_value=httpx.Response(
status_code=400,
content=b'{"type":"error","error":{"type":"invalid_request_error","message":"File id must have `file_` prefix."}}',
)
)
with pytest.raises(AnthropicError) as exc_info:
await handler.async_retrieve_file_content(
file_content_request={"file_id": "msgbatch_123"},
provider_config=AnthropicFilesConfig(),
litellm_params={"api_key": "sk-test"},
headers={},
logging_obj=Mock(),
client=client,
)
assert exc_info.value.status_code == 400
assert "file_" in str(exc_info.value)
def test_sync_retrieve_file_content_raises_on_http_error():
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.files.transformation import AnthropicFilesConfig
handler = BaseLLMHTTPHandler()
client = Mock(spec=HTTPHandler)
client.get = Mock(
return_value=httpx.Response(
status_code=404,
content=b'{"type":"error","error":{"type":"not_found_error","message":"not found"}}',
)
)
with pytest.raises(AnthropicError) as exc_info:
handler.retrieve_file_content(
file_content_request={"file_id": "file-abc"},
provider_config=AnthropicFilesConfig(),
litellm_params={"api_key": "sk-test"},
headers={},
logging_obj=Mock(),
client=client,
)
assert exc_info.value.status_code == 404
@pytest.mark.asyncio
async def test_handle_error_surfaces_provider_error_raised_while_aiohttp_streams_the_body(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.files.transformation import VertexAIFilesConfig
monkeypatch.setenv("NO_PROXY", "127.0.0.1")
monkeypatch.setenv("no_proxy", "127.0.0.1")
message: Final = "Batch input file line 3 is not valid JSON: Expecting value at column 1"
async def read_until_client_hangs_up(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
await reader.read()
writer.close()
async def body_with_bad_third_row():
yield b'{"custom_id": "r1"}\n'
raise VertexAIError(status_code=400, message=message)
server: Final = await asyncio.start_server(read_until_client_hangs_up, "127.0.0.1", 0)
port: Final = server.sockets[0].getsockname()[1]
handler: Final = AsyncHTTPHandler()
try:
async with server:
with pytest.raises(httpx.HTTPError) as sent:
await handler.post(f"http://127.0.0.1:{port}/upload", content=body_with_bad_third_row())
finally:
await handler.close()
with pytest.raises(VertexAIError) as surfaced:
BaseLLMHTTPHandler()._handle_error(e=sent.value, provider_config=VertexAIFilesConfig())
assert surfaced.value.status_code == 400
assert surfaced.value.message == message
def _runtime_error_caused_by(cause: Exception) -> RuntimeError:
try:
raise RuntimeError("upload bookkeeping failed") from cause
except RuntimeError as outer:
return outer
def test_handle_error_reports_its_own_failure_when_it_is_not_a_transport_error():
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.files.transformation import VertexAIFilesConfig
outer: Final = _runtime_error_caused_by(VertexAIError(status_code=400, message="inner provider error"))
with pytest.raises(VertexAIError) as surfaced:
BaseLLMHTTPHandler()._handle_error(e=outer, provider_config=VertexAIFilesConfig())
assert surfaced.value.status_code == 500
assert "upload bookkeeping failed" in surfaced.value.message
_FILE_CONTENT_WIF_ENV = {
"ANTHROPIC_FEDERATION_RULE_ID": "fdrl_llm_http_handler_seam",
"ANTHROPIC_ORGANIZATION_ID": "org-llm-http-handler-seam",
"ANTHROPIC_IDENTITY_TOKEN": "llm-http-handler-seam-inline-jwt",
}
class _BlockingWifPoster:
"""A token-endpoint poster that blocks until released, so the test can prove
the exchange ran off the event loop's own thread instead of freezing it."""
def __init__(self):
self.release = threading.Event()
self.thread_ids = []
def post(self, url, *, content, headers, timeout):
self.thread_ids.append(threading.get_ident())
self.release.wait(timeout=5)
return httpx.Response(
200,
json={
"access_token": "sk-ant-oat01-llm-http-handler-seam",
"token_type": "Bearer",
"expires_in": 3600,
},
)
@pytest.mark.asyncio
async def test_async_retrieve_file_content_wif_exchange_does_not_block_event_loop(monkeypatch):
"""Regression (Greptile P1): async_retrieve_file_content called the synchronous
validate_environment directly, so a cold WIF mint on this call site froze the
event loop until the exchange finished. It must resolve credentials through the
async facade instead."""
from litellm.llms.anthropic import common_utils as anthropic_common_utils
from litellm.llms.anthropic.files.transformation import AnthropicFilesConfig
from litellm.llms.anthropic.wif import aget_anthropic_wif_token, get_anthropic_wif_token
from litellm.llms.base_llm.auth.token_exchange import JwtBearerTokenExchangeEngine
for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
monkeypatch.delenv(name, raising=False)
for name, value in _FILE_CONTENT_WIF_ENV.items():
monkeypatch.setenv(name, value)
poster = _BlockingWifPoster()
engine = JwtBearerTokenExchangeEngine(poster=poster)
sync_calls = []
def sync_shim(litellm_params, api_base, model):
sync_calls.append(model)
return get_anthropic_wif_token(litellm_params, api_base, model, engine)
async def async_shim(litellm_params, api_base, model):
return await aget_anthropic_wif_token(litellm_params, api_base, model, engine)
monkeypatch.setattr(anthropic_common_utils, "get_anthropic_wif_token", sync_shim)
monkeypatch.setattr(anthropic_common_utils, "aget_anthropic_wif_token", async_shim)
handler = BaseLLMHTTPHandler()
client = Mock(spec=AsyncHTTPHandler)
client.get = AsyncMock(return_value=httpx.Response(status_code=200, content=b"file bytes"))
ticks = []
async def ticker():
for i in range(20):
await asyncio.sleep(0.005)
ticks.append(i)
ticker_task = asyncio.create_task(ticker())
await asyncio.sleep(0.02)
retrieve_task = asyncio.create_task(
handler.async_retrieve_file_content(
file_content_request={"file_id": "file-abc"},
provider_config=AnthropicFilesConfig(),
litellm_params={},
headers={},
logging_obj=Mock(),
client=client,
)
)
await asyncio.sleep(0.05)
# The ticker kept advancing while the token exchange was still blocked on
# poster.release, proving the exchange did not run inline on the event loop.
assert len(ticks) > 0
assert not retrieve_task.done()
poster.release.set()
await retrieve_task
await ticker_task
assert sync_calls == []
assert poster.thread_ids
assert poster.thread_ids[0] != threading.get_ident()
sent_headers = client.get.call_args.kwargs["headers"]
assert sent_headers["authorization"] == "Bearer sk-ant-oat01-llm-http-handler-seam"
_UPSTREAM_NOT_FOUND_BODY = {
"error": {
"message": "Response with id 'resp_abc' not found.",
"type": "invalid_request_error",
"param": None,
"code": None,
}
}
def _async_handler_returning(status_code: int, body: dict) -> AsyncHTTPHandler:
handler = AsyncHTTPHandler()
handler.client = httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(status_code, json=body))
)
return handler
def _sync_handler_returning(status_code: int, body: dict) -> HTTPHandler:
handler = HTTPHandler()
handler.client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(status_code, json=body)))
return handler
@pytest.mark.asyncio
async def test_aget_responses_surfaces_upstream_error_status_instead_of_500():
client = _async_handler_returning(404, _UPSTREAM_NOT_FOUND_BODY)
with pytest.raises(litellm.NotFoundError) as excinfo:
await litellm.aget_responses(
response_id="resp_abc",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
client=client,
)
assert excinfo.value.status_code == 404
assert "Response with id 'resp_abc' not found." in excinfo.value.message
def test_get_responses_surfaces_upstream_error_status_instead_of_500():
client = _sync_handler_returning(404, _UPSTREAM_NOT_FOUND_BODY)
with pytest.raises(litellm.NotFoundError) as excinfo:
litellm.get_responses(
response_id="resp_abc",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
client=client,
)
assert excinfo.value.status_code == 404
assert "Response with id 'resp_abc' not found." in excinfo.value.message
def test_list_input_items_surfaces_upstream_error_status():
client = _sync_handler_returning(404, _UPSTREAM_NOT_FOUND_BODY)
with pytest.raises(litellm.NotFoundError) as excinfo:
litellm.list_input_items(
response_id="resp_abc",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
client=client,
)
assert excinfo.value.status_code == 404
@pytest.mark.asyncio
async def test_alist_input_items_surfaces_upstream_error_status():
client = _async_handler_returning(404, _UPSTREAM_NOT_FOUND_BODY)
with pytest.raises(litellm.NotFoundError) as excinfo:
await litellm.alist_input_items(
response_id="resp_abc",
custom_llm_provider="azure",
api_base="https://test.openai.azure.com",
api_key="test-key",
api_version="2025-03-01-preview",
client=client,
)
assert excinfo.value.status_code == 404
@pytest.mark.asyncio
async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_request(monkeypatch):
"""Regression: after Bedrock rejects a replayed thinking block (400 invalid signature),
the strip-and-retry re-sign must not inherit attempt 1's SigV4 Authorization/X-Amz-Date;
reusing them over the new stripped body makes AWS return 403 SignatureDoesNotMatch."""
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
for env_var in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_SESSION_TOKEN", "AWS_PROFILE"):
monkeypatch.delenv(env_var, raising=False)
handler = BaseLLMHTTPHandler()
provider_config = AmazonAnthropicClaudeMessagesConfig()
litellm_params = GenericLiteLLMParams(
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
aws_region_name="us-east-1",
)
request_url = "https://bedrock-runtime.us-east-1.amazonaws.com/model/test-model/invoke"
request_body = {
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": 100,
"messages": [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "x", "signature": ""},
{"type": "text", "text": "ok"},
],
},
{"role": "user", "content": "continue"},
],
}
first_attempt_headers, signed_json_body = provider_config.sign_request(
headers={"Content-Type": "application/json"},
optional_params=dict(litellm_params),
request_data=request_body,
api_base=request_url,
api_key=None,
stream=False,
fake_stream=False,
model="test-model",
)
posts: list = []
invalid_signature_response = httpx.Response(
400,
text='{"message": "messages.1.content.0: Invalid `signature` in `thinking` block"}',
request=httpx.Request("POST", request_url),
)
ok_response = httpx.Response(200, json={"id": "msg_1"}, request=httpx.Request("POST", request_url))
class FakeAsyncClient:
async def post(self, url, headers, data, stream=False, logging_obj=None, timeout=None):
posts.append({"headers": dict(headers), "data": data})
return invalid_signature_response if len(posts) == 1 else ok_response
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
response = await handler._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=FakeAsyncClient(),
request_url=request_url,
headers=dict(first_attempt_headers),
signed_json_body=signed_json_body,
request_body=request_body,
stream=False,
logging_obj=logging_obj,
provider_config=provider_config,
litellm_params=litellm_params,
api_key=None,
model="test-model",
)
assert response.status_code == 200
assert len(posts) == 2
retry_payload = json.loads(posts[1]["data"])
retry_blocks = [
block
for message in retry_payload["messages"]
if isinstance(message.get("content"), list)
for block in message["content"]
]
assert retry_blocks and all(block["type"] != "thinking" for block in retry_blocks)
retry_authorization = posts[1]["headers"]["Authorization"]
assert retry_authorization.startswith("AWS4-HMAC-SHA256")
assert retry_authorization != first_attempt_headers["Authorization"]
def test_aws_signing_overrides_only_fills_missing_credentials():
from litellm.llms.custom_httpx.llm_http_handler import _aws_signing_overrides
overrides = _aws_signing_overrides(
{"temperature": 0.2, "aws_region_name": "us-west-2"},
{
"aws_role_name": "arn:aws:iam::000000000000:role/attributed",
"aws_session_name": "user-123",
"aws_region_name": "us-east-1",
"api_key": "not-an-aws-param",
},
)
assert dict(overrides) == {
"aws_role_name": "arn:aws:iam::000000000000:role/attributed",
"aws_session_name": "user-123",
}
class TestServerFulfilledToolsInRequest:
"""_server_fulfilled_tools_in_request gates the buffered (non-leaking) streaming
mode for server-fulfilled tools like headroom_retrieve."""
@staticmethod
def _logging_obj_with(callbacks):
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = callbacks
return logging_obj
def test_should_hold_back_when_callback_owns_tool_in_request(self):
from litellm.integrations.custom_logger import CustomLogger
class RetrievalCallback(CustomLogger):
server_fulfilled_tool_names = frozenset({"headroom_retrieve"})
tools = [
{"name": "Bash", "input_schema": {"type": "object"}},
{"name": "headroom_retrieve", "input_schema": {"type": "object"}},
]
assert BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
) == frozenset({"headroom_retrieve"})
def test_should_stream_live_when_tool_absent_from_request(self):
from litellm.integrations.custom_logger import CustomLogger
class RetrievalCallback(CustomLogger):
server_fulfilled_tool_names = frozenset({"headroom_retrieve"})
tools = [{"name": "Bash", "input_schema": {"type": "object"}}]
assert (
BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
)
== frozenset()
)
def test_should_stream_live_when_no_callback_declares_tool_names(self):
from litellm.integrations.custom_logger import CustomLogger
tools = [{"name": "headroom_retrieve", "input_schema": {"type": "object"}}]
assert (
BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
logging_obj=self._logging_obj_with([CustomLogger()]), tools=tools
)
== frozenset()
)
def test_should_stream_live_without_tools(self):
assert (
BaseLLMHTTPHandler._server_fulfilled_tools_in_request(logging_obj=self._logging_obj_with([]), tools=None)
== frozenset()
)
def test_interception_callbacks_declare_their_retrieval_tools(self):
from litellm.integrations.compression_interception.handler import (
LITELLM_CONTENT_RETRIEVE_TOOL_NAME,
CompressionInterceptionLogger,
)
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
HEADROOM_RETRIEVE_TOOL_NAME,
HeadroomGuardrail,
)
assert HeadroomGuardrail.server_fulfilled_tool_names == frozenset({HEADROOM_RETRIEVE_TOOL_NAME})
assert CompressionInterceptionLogger.server_fulfilled_tool_names == frozenset(
{LITELLM_CONTENT_RETRIEVE_TOOL_NAME}
)
def _make_stub_direct_vector_store_config(response):
from litellm.llms.base_llm.vector_store.transformation import (
BaseDirectVectorStoreConfig,
)
class StubDirectVectorStoreConfig(BaseDirectVectorStoreConfig):
def __init__(self):
super().__init__()
self.sync_calls = []
self.async_calls = []
def execute_search_vector_store_request(self, **kwargs):
self.sync_calls.append(kwargs)
return response
async def aexecute_search_vector_store_request(self, **kwargs):
self.async_calls.append(kwargs)
return response
return StubDirectVectorStoreConfig()
def test_vector_store_search_handler_direct_config_sync_skips_http():
handler = BaseLLMHTTPHandler()
stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []}
config = _make_stub_direct_vector_store_config(stub_response)
logging_obj = Mock()
with patch("litellm.llms.custom_httpx.llm_http_handler.get_httpx_client") as mock_get_client:
result = handler.vector_store_search_handler(
vector_store_id="vs_direct",
query="q",
vector_store_search_optional_params={"max_num_results": 4},
vector_store_provider_config=config,
custom_llm_provider="valkey",
litellm_params=GenericLiteLLMParams(valkey_host="localhost"),
logging_obj=logging_obj,
timeout=12.5,
_is_async=False,
)
assert result is stub_response
mock_get_client.assert_not_called()
assert len(config.sync_calls) == 1
call = config.sync_calls[0]
assert call["vector_store_id"] == "vs_direct"
assert call["query"] == "q"
assert call["timeout"] == 12.5
assert call["vector_store_search_optional_params"] == {"max_num_results": 4}
assert isinstance(call["litellm_params"], dict)
assert call["litellm_params"]["valkey_host"] == "localhost"
pre_call_args = logging_obj.pre_call.call_args.kwargs["additional_args"]
assert pre_call_args["query"] == "q"
assert pre_call_args["vector_store_id"] == "vs_direct"
@pytest.mark.asyncio
async def test_vector_store_search_handler_direct_config_async_skips_http():
handler = BaseLLMHTTPHandler()
stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []}
config = _make_stub_direct_vector_store_config(stub_response)
logging_obj = Mock()
with patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client") as mock_get_client:
result = await handler.vector_store_search_handler(
vector_store_id="vs_direct",
query=["q1", "q2"],
vector_store_search_optional_params={},
vector_store_provider_config=config,
custom_llm_provider="valkey",
litellm_params=GenericLiteLLMParams(valkey_host="localhost"),
logging_obj=logging_obj,
timeout=7.0,
_is_async=True,
)
assert result is stub_response
mock_get_client.assert_not_called()
assert len(config.async_calls) == 1
assert config.async_calls[0]["query"] == ["q1", "q2"]
assert config.async_calls[0]["litellm_params"]["valkey_host"] == "localhost"
assert config.async_calls[0]["timeout"] == 7.0
pre_call_args = logging_obj.pre_call.call_args.kwargs["additional_args"]
assert pre_call_args["query"] == ["q1", "q2"]
assert pre_call_args["vector_store_id"] == "vs_direct"
def _direct_vector_store_debug_logging_obj():
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
logging_obj = LitellmLogging(
model="valkey",
messages=[{"role": "user", "content": "q"}],
stream=False,
call_type="vector_store_search",
start_time=time.time(),
litellm_call_id="vs-debug-call-id",
function_id="vs-debug-function-id",
log_raw_request_response=True,
)
logging_obj.update_environment_variables(
model="valkey",
optional_params={"vector_store_id": "vs_direct", "query": "q"},
litellm_params={
"litellm_call_id": "vs-debug-call-id",
"vector_store_id": "vs_direct",
"litellm_request_debug": True,
"metadata": {"user_api_key_alias": "vs-test-key"},
"valkey_host": "valkey.internal",
"valkey_password": "sup3r-s3cret-valkey-pw",
"litellm_embedding_config": {"api_key": "sk-embedding-s3cret"},
},
)
return logging_obj
@pytest.mark.parametrize("is_async", [False, True])
def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, is_async):
"""Regression: an empty api_base made pre_call dump the whole model_call_details, so every
search shipped the stored valkey_password / embedding api_key into the raw_request metadata."""
handler = BaseLLMHTTPHandler()
stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []}
config = _make_stub_direct_vector_store_config(stub_response)
logging_obj = _direct_vector_store_debug_logging_obj()
with caplog.at_level(logging.DEBUG, logger=verbose_logger.name):
result = handler.vector_store_search_handler(
vector_store_id="vs_direct",
query="q",
vector_store_search_optional_params={"max_num_results": 4},
vector_store_provider_config=config,
custom_llm_provider="valkey",
litellm_params=GenericLiteLLMParams(
valkey_host="valkey.internal",
valkey_password="sup3r-s3cret-valkey-pw",
),
logging_obj=logging_obj,
_is_async=is_async,
)
if is_async:
result = asyncio.run(result)
assert result is stub_response
raw_request = logging_obj.model_call_details["litellm_params"]["metadata"]["raw_request"]
assert "sup3r-s3cret-valkey-pw" not in raw_request
assert "sk-embedding-s3cret" not in raw_request
assert "valkey://vs_direct" in raw_request
logged = "\n".join(record.getMessage() for record in caplog.records)
assert "sup3r-s3cret-valkey-pw" not in logged
assert "sk-embedding-s3cret" not in logged
@pytest.mark.asyncio
async def test_async_retrieve_batch_masks_presigned_auth_header_in_raw_request_log():
"""Regression: a pre-signed retrieve-batch request (Mistral, Bedrock) embeds its auth
header inside the transformed request, which pre_call logs verbatim as the raw request
body, so the provider key landed unmasked in raw_request_typed_dict and every
raw-request callback."""
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.mistral.batches.transformation import MistralBatchesConfig
provider_key = "mistral-s3cret-provider-key-123456"
job_payload = {
"id": "batch-1",
"input_files": ["file-1"],
"endpoint": "/v1/ocr",
"model": "mistral-ocr-latest",
"status": "SUCCESS",
"created_at": 1_757_400_000,
}
sent_requests = []
def _capture(request: httpx.Request) -> httpx.Response:
sent_requests.append(request)
return httpx.Response(200, json=job_payload)
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture))
logging_obj = LitellmLogging(
model="mistral/mistral-ocr-latest",
messages=[],
stream=False,
call_type="batch_retrieve",
start_time=time.time(),
litellm_call_id="batch-retrieve-call-id",
function_id="batch-retrieve-function-id",
log_raw_request_response=True,
)
logging_obj.update_environment_variables(
model="mistral/mistral-ocr-latest",
optional_params={},
litellm_params={"litellm_call_id": "batch-retrieve-call-id", "metadata": {}},
)
result = await BaseLLMHTTPHandler().retrieve_batch(
batch_id="batch-1",
litellm_params={"api_key": provider_key},
provider_config=MistralBatchesConfig(),
headers={},
api_base=None,
api_key=provider_key,
logging_obj=logging_obj,
_is_async=True,
client=client,
model="mistral/mistral-ocr-latest",
)
assert result.id == "batch-1"
assert sent_requests[0].headers["Authorization"] == f"Bearer {provider_key}"
raw_request_body = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_body"]
assert provider_key not in json.dumps(raw_request_body)
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_carries_deployment_vertex_location_for_pricing(monkeypatch):
"""
The proxy pre-creates the logging object before the router picks a deployment, so the
native /v1/messages path must copy the deployment's vertex_location into the logging
params it updates; otherwise cost resolution falls back to the environment and every
call on this surface prices with the regional uplift (#34393).
"""
import contextlib
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import (
Logging,
resolve_vertex_location_for_cost,
)
monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5")
monkeypatch.setattr(litellm, "vertex_location", None)
handler = BaseLLMHTTPHandler()
async def logging_obj_after_handler(generic_params):
logging_obj = Logging(
model="vertex_ai/claude-haiku-4-5@20251001",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="vertex-messages-location",
function_id="f",
)
logging_obj.update_environment_variables(
model="vertex_ai/claude-haiku-4-5@20251001",
user="",
optional_params={},
litellm_params={"api_base": ""},
custom_llm_provider="vertex_ai",
)
mock_config = Mock()
mock_config.avalidate_anthropic_messages_environment = AsyncMock(
return_value=({"authorization": "Bearer t"}, "https://us-east5-aiplatform.googleapis.com")
)
mock_config.transform_anthropic_messages_request = Mock(
return_value={"model": "claude-haiku-4-5@20251001", "messages": []}
)
with contextlib.suppress(Exception):
await handler.async_anthropic_messages_handler(
model="claude-haiku-4-5@20251001",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_provider_config=mock_config,
anthropic_messages_optional_request_params={"max_tokens": 10},
custom_llm_provider="vertex_ai",
litellm_params=generic_params,
logging_obj=logging_obj,
client=AsyncMock(),
kwargs={},
)
return logging_obj
global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global"))
assert global_deployment.litellm_params["vertex_location"] == "global"
assert (
resolve_vertex_location_for_cost(
custom_llm_provider="vertex_ai",
litellm_params=global_deployment.litellm_params,
optional_params=global_deployment.optional_params,
model="claude-haiku-4-5@20251001",
)
== "global"
)
unconfigured_deployment = await logging_obj_after_handler(GenericLiteLLMParams())
assert "vertex_location" not in unconfigured_deployment.litellm_params
_GENERIC_STREAM_SSE = (
b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,'
b'"model":"test-model","choices":[{"index":0,"delta":{"content":"hi"},'
b'"finish_reason":null}]}\n\n'
b"data: [DONE]\n\n"
)
def _generic_stream_upstream_response() -> httpx.Response:
return httpx.Response(
200,
headers={
"x-request-id": "generic-req-123",
"x-ratelimit-remaining-requests": "42",
},
content=_GENERIC_STREAM_SSE,
request=httpx.Request("POST", "https://fake-vllm.test/v1/chat/completions"),
)
def test_generic_http_handler_sync_streaming_forwards_provider_response_headers():
"""
Regression test for the generic BaseLLMHTTPHandler streaming path used by
~30 providers (deepseek, groq, hosted_vllm, databricks, openrouter, ...).
The sync `completion()` streaming branch builds the CustomStreamWrapper from
`make_sync_call`, which returns the upstream response headers alongside the
stream. Those headers must reach the caller as `llm_provider-*` entries in
`_hidden_params["additional_headers"]`, which is what the proxy merges into
the client-facing response headers.
"""
mock_client = Mock(spec=HTTPHandler)
mock_client.post = Mock(return_value=_generic_stream_upstream_response())
response = litellm.completion(
model="hosted_vllm/test-model",
messages=[{"role": "user", "content": "Hello"}],
api_base="https://fake-vllm.test/v1",
api_key="sk-test",
stream=True,
client=mock_client,
)
additional_headers = response._hidden_params["additional_headers"]
assert additional_headers["llm_provider-x-request-id"] == "generic-req-123"
assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42"
assert "".join([chunk.choices[0].delta.content or "" for chunk in response]) == "hi"
@pytest.mark.asyncio
async def test_generic_http_handler_async_streaming_forwards_provider_response_headers():
"""
Companion to the sync test above for `acompletion_stream_function`, which
builds its CustomStreamWrapper from `make_async_call_stream_helper`.
"""
mock_client = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(return_value=_generic_stream_upstream_response())
response = await litellm.acompletion(
model="hosted_vllm/test-model",
messages=[{"role": "user", "content": "Hello"}],
api_base="https://fake-vllm.test/v1",
api_key="sk-test",
stream=True,
client=mock_client,
)
additional_headers = response._hidden_params["additional_headers"]
assert additional_headers["llm_provider-x-request-id"] == "generic-req-123"
assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42"
collected = [chunk async for chunk in response]
assert "".join([chunk.choices[0].delta.content or "" for chunk in collected]) == "hi"
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure", "hosted_vllm", None])
def test_the_rust_responses_websocket_stays_on_python_with_the_switch_on(custom_llm_provider, monkeypatch):
from litellm.rust_bridge import configuration
configuration.reset_rust_configuration()
monkeypatch.setenv("LITELLM_RUST", "1")
assert _rust_responses_websocket_enabled(custom_llm_provider) is False
def test_a_plain_callback_does_not_advertise_a_pre_call_deployment_hook(monkeypatch):
from litellm.integrations.custom_logger import CustomLogger
class _PlainLogger(CustomLogger):
pass
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = []
monkeypatch.setattr(litellm, "callbacks", [])
assert _has_pre_call_deployment_hook(logging_obj) is False
monkeypatch.setattr(litellm, "callbacks", [_PlainLogger()])
assert _has_pre_call_deployment_hook(logging_obj) is False
def test_a_callback_that_overrides_the_deployment_hook_is_detected(monkeypatch):
from litellm.integrations.custom_logger import CustomLogger
class _DeploymentHookLogger(CustomLogger):
async def async_pre_call_deployment_hook(self, kwargs, call_type):
return None
class _InheritsTheHook(_DeploymentHookLogger):
pass
logging_obj = Mock()
logging_obj.dynamic_success_callbacks = []
monkeypatch.setattr(litellm, "callbacks", [_DeploymentHookLogger()])
assert _has_pre_call_deployment_hook(logging_obj) is True
monkeypatch.setattr(litellm, "callbacks", [_InheritsTheHook()])
assert _has_pre_call_deployment_hook(logging_obj) is True
monkeypatch.setattr(litellm, "callbacks", [])
logging_obj.dynamic_success_callbacks = [_DeploymentHookLogger()]
assert _has_pre_call_deployment_hook(logging_obj) is True
def test_only_callbacks_that_can_charge_a_frame_are_collected_for_ws_quota(monkeypatch):
from litellm.integrations.custom_logger import CustomLogger
class _PlainLogger(CustomLogger):
pass
class _QuotaLogger(CustomLogger):
async def enforce_project_io_token_quota_for_frame(self, *args, **kwargs):
return None
class _NotCallableAttribute:
enforce_project_io_token_quota_for_frame = "not a method"
plain, quota, decoy = _PlainLogger(), _QuotaLogger(), _NotCallableAttribute()
monkeypatch.setattr(litellm, "callbacks", [plain, decoy])
assert _collect_ws_project_quota_callbacks() == ()
monkeypatch.setattr(litellm, "callbacks", [plain, quota, decoy])
assert _collect_ws_project_quota_callbacks() == (quota,)
@pytest.mark.asyncio
async def test_async_rerank_records_llm_api_duration():
"""arerank must feed the httpx timing into the logging obj, so the proxy can emit
x-litellm-overhead-duration-ms / x-litellm-timing-* on /rerank."""
def handle(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "rerank-1",
"results": [{"index": 0, "relevance_score": 0.9}],
"meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 1}},
},
)
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
response = await litellm.arerank(
model="cohere/rerank-v3.5",
query="what is the capital of france",
documents=["paris", "berlin"],
top_n=1,
api_key="fake-key",
client=client,
)
assert response._hidden_params["litellm_overhead_time_ms"] is not None
assert response._hidden_params["_response_ms"] >= response._hidden_params["litellm_overhead_time_ms"]
class _JSONBodyVideoConfig(OpenAIVideoConfig):
def use_multipart_form_data(self) -> bool:
return False
def _video_create_call_kwargs(config, **optional_params):
return {
"model": "sora-2",
"prompt": "a cat surfing",
"video_generation_provider_config": config,
"video_generation_optional_request_params": {"seconds": "4", **optional_params},
"custom_llm_provider": "openai",
"litellm_params": GenericLiteLLMParams(api_key="sk-test", api_base="https://video.example/v1"),
"logging_obj": Mock(),
"timeout": 10.0,
}
def _capture_video_create_request(captured):
def respond(request):
captured["content_type"] = request.headers.get("content-type")
captured["body"] = request.content
return httpx.Response(
200,
json={"id": "video_123", "object": "video", "status": "queued", "created_at": 1712697600, "model": "sora-2"},
)
return respond
def _multipart_text_fields(content_type: str, body: bytes) -> dict:
boundary = content_type.split("boundary=")[1].encode()
return {
part.split(b'name="')[1].split(b'"')[0].decode(): part.partition(b"\r\n\r\n")[2].rstrip(b"\r\n-").decode()
for part in body.split(b"--" + boundary)
if b'name="' in part and b"filename=" not in part
}
def test_video_generation_without_file_sends_multipart_form_data():
"""Regression for #36493: the OpenAI SDK always sends /videos requests as
multipart/form-data, so OpenAI-compatible backends (SGLang Diffusion,
vLLM-Omni) reject the JSON body LiteLLM used to send when no
input_reference file was attached."""
captured = {}
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured))))
result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(OpenAIVideoConfig()))
assert captured["content_type"].startswith("multipart/form-data")
assert _multipart_text_fields(captured["content_type"], captured["body"]) == {
"model": "sora-2",
"prompt": "a cat surfing",
"seconds": "4",
}
assert result.status == "queued"
@pytest.mark.asyncio
async def test_async_video_generation_without_file_sends_multipart_form_data():
captured = {}
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture_video_create_request(captured)))
result = await BaseLLMHTTPHandler().async_video_generation_handler(
client=client, **_video_create_call_kwargs(OpenAIVideoConfig())
)
assert captured["content_type"].startswith("multipart/form-data")
assert _multipart_text_fields(captured["content_type"], captured["body"]) == {
"model": "sora-2",
"prompt": "a cat surfing",
"seconds": "4",
}
assert result.status == "queued"
def test_azure_video_generation_without_file_sends_multipart_form_data():
"""AzureVideoConfig subclasses OpenAIVideoConfig, so it inherits the
file-less multipart behavior. Azure's /openai/v1/videos surface is
OpenAI-SDK-compatible (the SDK sends multipart there too), so this is
intentional; lock it so the inherited flip can't silently regress to JSON."""
assert AzureVideoConfig().use_multipart_form_data() is True
captured = {}
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured))))
result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(AzureVideoConfig()))
assert captured["content_type"].startswith("multipart/form-data")
assert _multipart_text_fields(captured["content_type"], captured["body"]) == {
"model": "sora-2",
"prompt": "a cat surfing",
"seconds": "4",
}
assert result.status == "queued"
def test_video_generation_json_provider_keeps_json_body():
captured = {}
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured))))
result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(_JSONBodyVideoConfig()))
assert captured["content_type"] == "application/json"
assert json.loads(captured["body"]) == {"model": "sora-2", "prompt": "a cat surfing", "seconds": "4"}
assert result.status == "queued"
def test_video_generation_with_input_reference_keeps_file_multipart():
captured = {}
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured))))
result = BaseLLMHTTPHandler().video_generation_handler(
client=client,
**_video_create_call_kwargs(OpenAIVideoConfig(), input_reference=b"\x89PNG\r\n\x1a\nfakepng"),
)
assert captured["content_type"].startswith("multipart/form-data")
assert b'name="input_reference"' in captured["body"]
assert b'filename="input_reference.png"' in captured["body"]
assert _multipart_text_fields(captured["content_type"], captured["body"]) == {
"model": "sora-2",
"prompt": "a cat surfing",
"seconds": "4",
}
assert result.status == "queued"
AZURE_AI_BASE = "https://myfoundry.services.ai.azure.com"
AZURE_AI_CHAT_COMPLETIONS_URL = f"{AZURE_AI_BASE}/models/chat/completions"
def _a_tool_with_an_unsupported_field() -> dict:
return {
"type": "function",
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
"strict": True,
}
A_COMPLETION = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "grok-3",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
TOOL_LEVEL_REJECTION = "Extra inputs are not permitted: tools[0].strict"
UNRELATED_REJECTION = "Extra inputs are not permitted: temperature"
A_REJECTION_THE_PROVIDER_CANNOT_FIX = "The model is not available in this region"
class _RecordedAzureAI:
def __init__(self, responses: list[httpx.Response]) -> None:
self._responses = responses
self.bodies: list[dict] = []
def __call__(self, request: httpx.Request) -> httpx.Response:
self.bodies.append(json.loads(request.content))
return self._responses[min(len(self.bodies) - 1, len(self._responses) - 1)]
@pytest.fixture
def httpx_transport(monkeypatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
def _rejection(message: str) -> httpx.Response:
return httpx.Response(422, json={"error": {"message": message}})
def _call_azure_ai(recorder: _RecordedAzureAI, **overrides):
import respx
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
return litellm.completion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
**overrides,
)
def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried():
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
response = _call_azure_ai(recorder)
assert len(recorder.bodies) == 2
assert recorder.bodies[0]["tools"][0]["strict"] is True
assert "strict" not in recorder.bodies[1]["tools"][0]
assert response.choices[0].message.content == "sent"
def test_the_retry_changes_only_the_field_the_provider_named():
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
_call_azure_ai(recorder)
first, second = recorder.bodies
assert second["messages"] == first["messages"]
assert second["model"] == first["model"]
assert second["tools"][0]["function"] == first["tools"][0]["function"]
def test_a_provider_that_keeps_rejecting_is_not_retried_forever():
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
with pytest.raises(litellm.BadRequestError) as raised:
_call_azure_ai(recorder)
assert len(recorder.bodies) == 2
assert raised.value.status_code == 422
def test_a_rejection_the_provider_cannot_fix_is_not_retried_at_all():
recorder = _RecordedAzureAI([_rejection(A_REJECTION_THE_PROVIDER_CANNOT_FIX)])
with pytest.raises(litellm.BadRequestError):
_call_azure_ai(recorder)
assert len(recorder.bodies) == 1
def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was_asked_for():
recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION)])
with pytest.raises(litellm.BadRequestError):
_call_azure_ai(recorder)
assert len(recorder.bodies) == 1
def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for():
recorder = _RecordedAzureAI(
[_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
response = _call_azure_ai(recorder, drop_params=True)
assert len(recorder.bodies) == 2
assert response.choices[0].message.content == "sent"
@pytest.mark.asyncio
async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_async_path(
httpx_transport,
):
import respx
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
response = await litellm.acompletion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
)
assert len(recorder.bodies) == 2
assert recorder.bodies[0]["tools"][0]["strict"] is True
assert "strict" not in recorder.bodies[1]["tools"][0]
assert response.choices[0].message.content == "sent"
@pytest.mark.asyncio
async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_async_path(
httpx_transport,
):
import respx
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
with pytest.raises(litellm.BadRequestError):
await litellm.acompletion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
)
assert len(recorder.bodies) == 2
def _async_client_returning(response: Mock) -> AsyncMock:
client = AsyncMock(spec=AsyncHTTPHandler)
client.post.return_value = response
return client
@pytest.mark.asyncio
async def test_create_file_async_awaits_the_provider_credential_hook_instead_of_blocking():
provider_config = Mock(spec=BaseFilesConfig)
provider_config.validate_environment.side_effect = AssertionError("sync validate_environment ran on the event loop")
provider_config.avalidate_environment = AsyncMock(return_value={"x-api-key": "federated"})
provider_config.get_complete_file_url.return_value = "https://files.example/v1/files"
provider_config.transform_create_file_request.return_value = {"file": ("batch.jsonl", b"{}", "application/jsonl")}
file_object = object()
provider_config.transform_create_file_response.return_value = file_object
client = _async_client_returning(Mock(spec=httpx.Response))
result = await BaseLLMHTTPHandler().create_file(
create_file_data={"file": b"{}", "purpose": "batch"},
litellm_params={},
provider_config=provider_config,
headers={},
api_base=None,
api_key=None,
logging_obj=Mock(),
_is_async=True,
client=client,
)
assert result is file_object
provider_config.validate_environment.assert_not_called()
provider_config.avalidate_environment.assert_awaited_once()
assert client.post.call_args.kwargs["headers"] == {"x-api-key": "federated"}
assert client.post.call_args.kwargs["url"] == "https://files.example/v1/files"
@pytest.mark.asyncio
async def test_create_batch_async_validates_credentials_off_the_event_loop():
provider_config = Mock(spec=BaseBatchesConfig)
provider_config.validate_environment.side_effect = lambda **_: {"x-validated-on": str(threading.get_ident())}
provider_config.get_complete_batch_url.return_value = "https://batches.example/v1/messages/batches"
provider_config.transform_create_batch_request.return_value = {"requests": []}
batch = object()
provider_config.transform_create_batch_response.return_value = batch
client = _async_client_returning(Mock(spec=httpx.Response))
result = await BaseLLMHTTPHandler().create_batch(
create_batch_data={"input_file_id": "file_1", "endpoint": "/v1/chat/completions", "completion_window": "24h"},
litellm_params={},
provider_config=provider_config,
headers={},
api_base=None,
api_key=None,
logging_obj=Mock(),
_is_async=True,
client=client,
model="claude-sonnet-4-5",
)
assert result is batch
validated_on = client.post.call_args.kwargs["headers"]["x-validated-on"]
assert validated_on != str(threading.get_ident())
assert client.post.call_args.kwargs["url"] == "https://batches.example/v1/messages/batches"
@pytest.mark.asyncio
async def test_create_file_says_which_setting_is_missing_when_the_provider_resolves_no_url():
"""A provider that cannot work out where its files endpoint lives returns no URL, which used to
be posted as-is: the caller saw an httpx error about an invalid URL and no mention of api_base."""
provider_config = Mock(spec=BaseFilesConfig)
provider_config.avalidate_environment = AsyncMock(return_value={})
provider_config.get_complete_file_url.return_value = None
client = _async_client_returning(Mock(spec=httpx.Response))
with pytest.raises(ValueError, match="api_base is required for create_file"):
await BaseLLMHTTPHandler().create_file(
create_file_data={"file": b"{}", "purpose": "batch"},
litellm_params={},
provider_config=provider_config,
headers={},
api_base=None,
api_key=None,
logging_obj=Mock(),
_is_async=True,
client=client,
)
client.post.assert_not_called()
provider_config.transform_create_file_request.assert_not_called()
@pytest.mark.asyncio
async def test_create_batch_says_which_setting_is_missing_when_the_provider_resolves_no_url():
"""Same on the batches path, which resolves its URL the same way."""
provider_config = Mock(spec=BaseBatchesConfig)
provider_config.validate_environment.return_value = {}
provider_config.get_complete_batch_url.return_value = None
client = _async_client_returning(Mock(spec=httpx.Response))
with pytest.raises(ValueError, match="api_base is required for create_batch"):
await BaseLLMHTTPHandler().create_batch(
create_batch_data={"input_file_id": "file_1", "endpoint": "/v1/chat/completions", "completion_window": "24h"},
litellm_params={},
provider_config=provider_config,
headers={},
api_base=None,
api_key=None,
logging_obj=Mock(),
_is_async=True,
client=client,
model="claude-sonnet-4-5",
)
client.post.assert_not_called()
provider_config.transform_create_batch_request.assert_not_called()
CONTAINER_NOT_FOUND_BODY = {
"error": {
"message": "Container with id 'cntr_gone' not found.",
"type": "invalid_request_error",
"param": None,
"code": None,
}
}
INVALID_API_KEY_BODY = {
"error": {
"message": "Incorrect API key provided: sk-proj-***. You can find your API key at https://platform.openai.com/account/api-keys.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_api_key",
},
"status": 401,
}
CONTAINER_LIST_BODY = {
"object": "list",
"data": [{"id": "cntr_a", "object": "container", "created_at": 1, "status": "running", "name": "a"}],
"first_id": "cntr_a",
"last_id": "cntr_a",
"has_more": True,
}
def _container_sync_client(response: httpx.Response) -> HTTPHandler:
client = HTTPHandler()
client.client = httpx.Client(transport=httpx.MockTransport(lambda _request: response))
return client
def _container_async_client(response: httpx.Response) -> AsyncHTTPHandler:
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _request: response))
return client
def test_container_retrieve_handler_raises_upstream_error_status_and_message():
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
with pytest.raises(BaseLLMException) as exc_info:
BaseLLMHTTPHandler().container_retrieve_handler(
container_id="cntr_gone",
container_provider_config=OpenAIContainerConfig(),
litellm_params=GenericLiteLLMParams(api_key="sk-test"),
logging_obj=Mock(),
client=_container_sync_client(httpx.Response(404, json=CONTAINER_NOT_FOUND_BODY)),
)
assert exc_info.value.status_code == 404
assert exc_info.value.message == "Container with id 'cntr_gone' not found."
@pytest.mark.asyncio
async def test_async_container_list_handler_raises_upstream_error_status_and_message():
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
with pytest.raises(BaseLLMException) as exc_info:
await BaseLLMHTTPHandler().async_container_list_handler(
container_provider_config=OpenAIContainerConfig(),
litellm_params=GenericLiteLLMParams(api_key="sk-rejected"),
logging_obj=Mock(),
client=_container_async_client(httpx.Response(401, json=INVALID_API_KEY_BODY)),
)
assert exc_info.value.status_code == 401
assert exc_info.value.message == INVALID_API_KEY_BODY["error"]["message"]
@pytest.mark.asyncio
async def test_async_container_list_handler_transforms_success_response():
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
response = await BaseLLMHTTPHandler().async_container_list_handler(
container_provider_config=OpenAIContainerConfig(),
litellm_params=GenericLiteLLMParams(api_key="sk-test"),
logging_obj=Mock(),
limit=1,
client=_container_async_client(httpx.Response(200, json=CONTAINER_LIST_BODY)),
)
assert [container.id for container in response.data] == ["cntr_a"]
assert response.has_more is True
class _TransformRecordingConfig(BaseConfig):
def __init__(self, transform_async: bool):
self.transform_async = transform_async
self.transform_calls = []
self.sign_threads = []
@property
def uses_async_transform_request(self) -> bool:
return self.transform_async
def get_supported_openai_params(self, model):
return []
def map_openai_params(self, non_default_params, optional_params, model, drop_params):
return optional_params
def validate_environment(
self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None
):
return {}
def transform_request(self, model, messages, optional_params, litellm_params, headers):
self.transform_calls.append("sync")
return {"transformed_by": "sync"}
async def async_transform_request(self, model, messages, optional_params, litellm_params, headers):
self.transform_calls.append("async")
return {"transformed_by": "async"}
def sign_request(
self, headers, optional_params, request_data, api_base, api_key=None, model=None, stream=None, fake_stream=None
):
self.sign_threads.append(threading.current_thread())
return headers, None
def transform_response(
self,
model,
raw_response,
model_response,
logging_obj,
request_data,
messages,
optional_params,
litellm_params,
encoding,
api_key=None,
json_mode=None,
):
model_response.choices[0].message.content = raw_response.json()["transformed_by"]
return model_response
def get_error_class(self, error_message, status_code, headers):
return BaseLLMException(status_code=status_code, message=error_message, headers=headers)
def get_model_response_iterator(self, streaming_response, sync_stream, json_mode=False):
return litellm.OpenAIGPTConfig().get_model_response_iterator(
streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode
)
def _start_async_completion(config, logging_obj=None):
captured = {}
def handle(request):
captured["body"] = json.loads(request.content)
return httpx.Response(200, json=captured["body"])
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
pending = BaseLLMHTTPHandler().completion(
model="stub-model",
messages=[{"role": "user", "content": "hi"}],
api_base="https://llm.example/v1/chat",
custom_llm_provider="openai",
model_response=ModelResponse(),
encoding=None,
logging_obj=logging_obj if logging_obj is not None else Mock(dynamic_success_callbacks=None, model_call_details={}),
optional_params={},
timeout=10.0,
litellm_params={},
acompletion=True,
client=client,
provider_config=config,
)
return pending, captured
async def test_completion_awaits_async_transform_request_when_config_opts_in():
config = _TransformRecordingConfig(transform_async=True)
pending, captured = _start_async_completion(config)
assert config.transform_calls == []
response = await pending
assert config.transform_calls == ["async"]
assert captured["body"] == {"transformed_by": "async"}
assert response.choices[0].message.content == "async"
async def test_completion_signs_and_logs_off_the_event_loop_after_the_async_transform():
config = _TransformRecordingConfig(transform_async=True)
loop_thread = threading.current_thread()
pre_call_threads = []
logging_obj = Mock(dynamic_success_callbacks=None, model_call_details={})
logging_obj.pre_call.side_effect = lambda **kwargs: pre_call_threads.append(threading.current_thread())
pending, captured = _start_async_completion(config, logging_obj)
response = await pending
assert response.choices[0].message.content == "async"
assert captured["body"] == {"transformed_by": "async"}
assert config.sign_threads and all(thread is not loop_thread for thread in config.sign_threads)
assert pre_call_threads and all(thread is not loop_thread for thread in pre_call_threads)
assert not any(thread.name.startswith("aws-signing") for thread in config.sign_threads + pre_call_threads)
class _AWSTransformRecordingConfig(SignsRequestsWithAWS, _TransformRecordingConfig):
pass
async def test_completion_signs_aws_configs_on_the_aws_signing_pool_after_the_async_transform():
config = _AWSTransformRecordingConfig(transform_async=True)
pre_call_threads = []
logging_obj = Mock(dynamic_success_callbacks=None, model_call_details={})
logging_obj.pre_call.side_effect = lambda **kwargs: pre_call_threads.append(threading.current_thread())
pending, captured = _start_async_completion(config, logging_obj)
response = await pending
assert response.choices[0].message.content == "async"
assert captured["body"] == {"transformed_by": "async"}
assert config.sign_threads and all(thread.name.startswith("aws-signing") for thread in config.sign_threads)
assert pre_call_threads and all(thread.name.startswith("aws-signing") for thread in pre_call_threads)
async def test_completion_keeps_sync_transform_request_before_returning_by_default():
config = _TransformRecordingConfig(transform_async=False)
pending, captured = _start_async_completion(config)
assert config.transform_calls == ["sync"]
response = await pending
assert config.transform_calls == ["sync"]
assert captured["body"] == {"transformed_by": "sync"}
assert response.choices[0].message.content == "sync"
def _sse_echoing_transformed_by(request):
transformed_by = json.loads(request.content)["transformed_by"]
chunk = {
"id": "chatcmpl-1",
"object": "chat.completion.chunk",
"created": 1,
"model": "stub-model",
"choices": [{"index": 0, "delta": {"content": transformed_by}, "finish_reason": None}],
}
return httpx.Response(
200,
content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode(),
headers={"content-type": "text/event-stream"},
request=request,
)
def _streaming_logging_obj():
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="stub-model",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="acompletion",
start_time=time.time(),
litellm_call_id="async-transform-stream",
function_id="f",
)
logging_obj.update_environment_variables(
model="stub-model", user="", optional_params={}, litellm_params={}, custom_llm_provider="openai"
)
return logging_obj
async def test_completion_streams_after_the_async_transform_request():
config = _TransformRecordingConfig(transform_async=True)
loop_thread = threading.current_thread()
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(_sse_echoing_transformed_by))
stream = await BaseLLMHTTPHandler().completion(
model="stub-model",
messages=[{"role": "user", "content": "hi"}],
api_base="https://llm.example/v1/chat",
custom_llm_provider="openai",
model_response=ModelResponse(),
encoding=None,
logging_obj=_streaming_logging_obj(),
optional_params={},
timeout=10.0,
litellm_params={},
acompletion=True,
stream=True,
client=client,
provider_config=config,
)
collected = [chunk async for chunk in stream]
assert config.transform_calls == ["async"]
assert config.sign_threads and all(thread is not loop_thread for thread in config.sign_threads)
assert "".join(chunk.choices[0].delta.content or "" for chunk in collected) == "async"
class _IteratorRecordingConfig(_TransformRecordingConfig):
def __init__(self):
super().__init__(transform_async=False)
self.iterator_requests = []
def get_model_response_iterator(self, streaming_response, sync_stream, json_mode=False):
self.iterator_requests.append({"sync_stream": sync_stream, "json_mode": json_mode})
return super().get_model_response_iterator(streaming_response, sync_stream, json_mode)
def _json_mode_stream(config, **transport):
return BaseLLMHTTPHandler().completion(
model="stub-model",
messages=[{"role": "user", "content": "hi"}],
api_base="https://llm.example/v1/chat",
custom_llm_provider="openai",
model_response=ModelResponse(),
encoding=None,
logging_obj=_streaming_logging_obj(),
optional_params={"json_mode": True},
timeout=10.0,
litellm_params={},
stream=True,
provider_config=config,
**transport,
)
def _streamed_text(chunks):
return "".join(chunk.choices[0].delta.content or "" for chunk in chunks)
async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_request_hook_only_when_sync():
config = _IteratorRecordingConfig()
sync_client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_sse_echoing_transformed_by)))
async_client = AsyncHTTPHandler()
async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_sse_echoing_transformed_by))
sync_chunks = list(_json_mode_stream(config, acompletion=False, client=sync_client))
async_chunks = [chunk async for chunk in await _json_mode_stream(config, acompletion=True, client=async_client)]
assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync")
assert config.iterator_requests == [
{"sync_stream": True, "json_mode": True},
{"sync_stream": False, "json_mode": False},
]
class _ImageEditRecordingConfig(BaseImageEditConfig):
def __init__(self):
self.transform_calls = []
def get_supported_openai_params(self, model):
return []
def map_openai_params(self, image_edit_optional_params, model, drop_params):
return dict(image_edit_optional_params)
def validate_environment(self, headers, model, api_key=None, litellm_params=None, api_base=None):
return {}
def get_complete_url(self, model, api_base, litellm_params):
return "https://images.example/v1/edits"
def use_multipart_form_data(self):
return False
def transform_image_edit_request(
self, model, prompt, image, image_edit_optional_request_params, litellm_params, headers
):
self.transform_calls.append("sync")
return {"transformed_by": "sync"}, []
async def async_transform_image_edit_request(
self, model, prompt, image, image_edit_optional_request_params, litellm_params, headers
):
self.transform_calls.append("async")
return {"transformed_by": "async"}, []
def transform_image_edit_response(self, model, raw_response, logging_obj):
return ImageResponse(data=[ImageObject(b64_json=raw_response.json()["transformed_by"])])
def _echo_json_transport(captured):
def handle(request):
captured["body"] = json.loads(request.content)
return httpx.Response(200, json=captured["body"])
return httpx.MockTransport(handle)
async def test_async_image_edit_handler_awaits_the_async_transform():
config = _ImageEditRecordingConfig()
captured = {}
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=_echo_json_transport(captured))
response = await BaseLLMHTTPHandler().async_image_edit_handler(
model="edit-model",
image=b"raw-image",
prompt="add a hat",
image_edit_provider_config=config,
image_edit_optional_request_params={},
custom_llm_provider="openai",
litellm_params=GenericLiteLLMParams(),
logging_obj=Mock(),
timeout=10.0,
client=client,
)
assert config.transform_calls == ["async"]
assert captured["body"] == {"transformed_by": "async"}
assert response.data[0].b64_json == "async"
def test_image_edit_handler_keeps_the_sync_transform():
config = _ImageEditRecordingConfig()
captured = {}
client = HTTPHandler()
client.client = httpx.Client(transport=_echo_json_transport(captured))
response = BaseLLMHTTPHandler().image_edit_handler(
model="edit-model",
image=b"raw-image",
prompt="add a hat",
image_edit_provider_config=config,
image_edit_optional_request_params={},
custom_llm_provider="openai",
litellm_params=GenericLiteLLMParams(),
logging_obj=Mock(),
timeout=10.0,
client=client,
)
assert config.transform_calls == ["sync"]
assert captured["body"] == {"transformed_by": "sync"}
assert response.data[0].b64_json == "sync"
class _ScriptedClientWebSocket(_FakeClientWebSocket):
def __init__(self, messages: list[str], last_event_type: str) -> None:
super().__init__()
self._messages: Final = list(messages)
self._last_event_type: Final = last_event_type
self._backend_done: Final = asyncio.Event()
async def receive_text(self) -> str:
if self._messages:
return self._messages.pop(0)
await asyncio.wait_for(self._backend_done.wait(), timeout=5)
raise RuntimeError("client went away")
async def send_text(self, payload: str) -> None:
await super().send_text(payload)
if json.loads(payload).get("type") == self._last_event_type:
self._backend_done.set()
def sent_events(self) -> list[dict[str, object]]:
return [json.loads(payload) for name, payload in self.events if name == "send_text"]
@pytest.mark.asyncio
async def test_async_realtime_bridges_a_transcription_session_through_the_provider_backend():
import websockets.exceptions # noqa: F401 # binds the submodule so async_realtime's except clause resolves, as in the proxy process
from datetime import timedelta
from google.cloud.speech_v2.types import (
RecognitionResponseMetadata,
SpeechRecognitionAlternative,
StreamingRecognitionResult,
StreamingRecognizeResponse,
)
from litellm.llms.vertex_ai.audio_transcription.realtime_backend import SpeechStreamingBackend
from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig
def google_response(transcript: str, is_final: bool, billed: float) -> StreamingRecognizeResponse:
return StreamingRecognizeResponse(
results=[
StreamingRecognitionResult(
alternatives=[SpeechRecognitionAlternative(transcript=transcript)], is_final=is_final
)
],
metadata=RecognitionResponseMetadata(total_billed_duration=timedelta(seconds=billed)),
)
class FakeTransport:
async def close(self) -> None:
return None
class FakeSpeechClient:
transport = FakeTransport()
def __init__(self) -> None:
self.requests: Final[list[object]] = []
async def streaming_recognize(self, requests=None):
return self._respond(requests)
async def _respond(self, requests):
script = [google_response("four score", False, 0.0), google_response("Four score and seven", True, 2.0)]
async for request in requests:
self.requests.append(request)
if request.audio and script:
yield script.pop(0)
speech_client = FakeSpeechClient()
async def resolve_access_token() -> str:
return "token"
provider_config = VertexChirpRealtimeConfig(
resolve_access_token=resolve_access_token,
project="proj-1",
location="us",
backend_factory=lambda target: SpeechStreamingBackend(
target, client_factory=lambda target, access_token: speech_client
),
)
audio = base64.b64encode(b"\x00\x01" * 800).decode()
client_ws = _ScriptedClientWebSocket(
[
json.dumps(
{
"type": "session.update",
"session": {
"type": "transcription",
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "chirp_3", "language": "en"},
"turn_detection": {"type": "server_vad"},
}
},
},
}
),
json.dumps({"type": "input_audio_buffer.append", "audio": audio}),
json.dumps({"type": "input_audio_buffer.append", "audio": audio}),
json.dumps({"type": "input_audio_buffer.commit"}),
],
last_event_type="conversation.item.input_audio_transcription.completed",
)
logging_obj = Mock()
logging_obj.litellm_trace_id = "trace_1"
logging_obj.model_call_details = {}
logging_obj.dispatch_success_handlers = AsyncMock()
logging_obj.dispatch_failure_handlers = AsyncMock()
handler = BaseLLMHTTPHandler()
with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(side_effect=AssertionError("dialed a websocket"))) as dial:
await handler.async_realtime(
model="chirp_3",
websocket=client_ws,
logging_obj=logging_obj,
provider_config=provider_config,
headers={},
query_params={"model": "chirp_3", "intent": "transcription"},
)
dial.assert_not_awaited()
events = client_ws.sent_events()
assert [event["type"] for event in events] == [
"session.created",
"session.updated",
"input_audio_buffer.speech_started",
"conversation.item.input_audio_transcription.delta",
"conversation.item.input_audio_transcription.delta",
"input_audio_buffer.speech_stopped",
"conversation.item.input_audio_transcription.completed",
]
assert events[0]["session"]["audio"]["input"]["transcription"] == {"model": "chirp_3"}
assert events[1]["session"]["audio"]["input"] == {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "chirp_3", "language": "en-US"},
"turn_detection": {"type": "server_vad"},
}
assert [event["delta"] for event in events[3:5]] == ["four score", " and seven"]
assert events[6]["transcript"] == "Four score and seven"
assert events[6]["usage"] == {"type": "duration", "seconds": 2.0}
assert speech_client.requests[0].streaming_config.config.model == "chirp_3"
assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800]
@pytest.mark.asyncio
async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch):
"""A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword"""
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
followup_calls: list[dict[str, object]] = []
async def fake_aresponses(**kwargs: object) -> str:
followup_calls.append(kwargs)
return "followup-response"
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
request_kwargs: Final = {"prompt_cache_key": "thread-1", "metadata": {"user": "u1"}}
plan: Final = AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
optional_params={"prompt_cache_key": "thread-1"},
kwargs=dict(request_kwargs),
),
)
response: Final = await BaseLLMHTTPHandler()._execute_responses_agentic_plan(
plan=plan,
model="gpt-5",
response_api_optional_request_params={"prompt_cache_key": "thread-1"},
logging_obj=Mock(litellm_call_id="call-1"),
kwargs=dict(request_kwargs),
depth=0,
max_loops=3,
fingerprints=[],
fingerprint="fp",
callback=CustomLogger(),
)
assert response == "followup-response"
assert len(followup_calls) == 1
assert followup_calls[0]["prompt_cache_key"] == "thread-1"
assert followup_calls[0]["metadata"] == {"user": "u1"}
assert followup_calls[0]["_agentic_loop_depth"] == 1
@pytest.mark.asyncio
async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_stale_kwargs_copy(monkeypatch):
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
followup_calls: list[dict[str, object]] = []
async def fake_aresponses(**kwargs: object) -> str:
followup_calls.append(kwargs)
return "followup-response"
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
await BaseLLMHTTPHandler()._execute_responses_agentic_plan(
plan=AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
optional_params={"prompt_cache_key": "from-plan-params"},
kwargs={"prompt_cache_key": "stale-copy"},
),
),
model="gpt-5",
response_api_optional_request_params={"prompt_cache_key": "from-request"},
logging_obj=Mock(litellm_call_id="call-1"),
kwargs={},
depth=0,
max_loops=3,
fingerprints=[],
fingerprint="fp",
callback=CustomLogger(),
)
assert followup_calls[0]["prompt_cache_key"] == "from-plan-params"
@pytest.mark.asyncio
async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch):
"""A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword"""
from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
followup_calls: list[dict[str, object]] = []
async def fake_acompletion(**kwargs: object) -> str:
followup_calls.append(kwargs)
return "followup-response"
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
request_kwargs: Final = {"temperature": 0.2, "api_base": "https://a", "model": "gpt-5"}
plan: Final = AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
optional_params={"temperature": 0.2},
kwargs=dict(request_kwargs),
),
)
response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan(
plan=plan,
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
optional_params={"temperature": 0.2},
kwargs=dict(request_kwargs),
custom_llm_provider="openai",
depth=0,
max_loops=3,
fingerprints=[],
fingerprint="fp",
)
assert response == "followup-response"
assert len(followup_calls) == 1
assert followup_calls[0]["temperature"] == 0.2
assert followup_calls[0]["api_base"] == "https://a"
assert followup_calls[0]["model"] == "openai/gpt-5"
_UPSTREAM_HEADERS: Final = {"x-request-id": "req_upstream", "x-ratelimit-remaining-requests": "41"}
def _assert_upstream_headers_recorded(response) -> None:
assert response._hidden_params["headers"]["x-request-id"] == "req_upstream"
assert response._hidden_params["additional_headers"]["llm_provider-x-request-id"] == "req_upstream"
assert response._hidden_params["additional_headers"]["x-ratelimit-remaining-requests"] == "41"
def _json_with_upstream_headers(payload: dict) -> httpx.MockTransport:
return httpx.MockTransport(lambda request: httpx.Response(200, json=payload, headers=_UPSTREAM_HEADERS))
def _binary_with_upstream_headers() -> httpx.MockTransport:
return httpx.MockTransport(
lambda request: httpx.Response(
200, content=b"audio-bytes", headers={**_UPSTREAM_HEADERS, "content-type": "audio/mpeg"}
)
)
def test_audio_transcriptions_records_upstream_response_headers():
client = HTTPHandler(client=httpx.Client(transport=_json_with_upstream_headers({"text": "transcribed"})))
response = BaseLLMHTTPHandler().audio_transcriptions(
client=client,
atranscription=False,
**_json_transcription_call_kwargs(_JSONBodyAudioTranscriptionConfig()),
)
_assert_upstream_headers_recorded(response)
@pytest.mark.asyncio
async def test_async_audio_transcriptions_records_upstream_response_headers():
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=_json_with_upstream_headers({"text": "transcribed"}))
response = await BaseLLMHTTPHandler().async_audio_transcriptions(
client=client,
**_json_transcription_call_kwargs(_JSONBodyAudioTranscriptionConfig()),
)
_assert_upstream_headers_recorded(response)
def _image_edit_call_kwargs() -> dict:
return {
"model": "edit-model",
"image": b"raw-image",
"prompt": "add a hat",
"image_edit_provider_config": _ImageEditRecordingConfig(),
"image_edit_optional_request_params": {},
"custom_llm_provider": "openai",
"litellm_params": GenericLiteLLMParams(),
"logging_obj": Mock(),
"timeout": 10.0,
}
def test_image_edit_handler_records_upstream_response_headers():
client = HTTPHandler()
client.client = httpx.Client(transport=_json_with_upstream_headers({"transformed_by": "sync"}))
response = BaseLLMHTTPHandler().image_edit_handler(client=client, **_image_edit_call_kwargs())
_assert_upstream_headers_recorded(response)
@pytest.mark.asyncio
async def test_async_image_edit_handler_records_upstream_response_headers():
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=_json_with_upstream_headers({"transformed_by": "async"}))
response = await BaseLLMHTTPHandler().async_image_edit_handler(client=client, **_image_edit_call_kwargs())
_assert_upstream_headers_recorded(response)
class _HeaderImageGenerationConfig(BaseImageGenerationConfig):
def get_supported_openai_params(self, model):
return []
def map_openai_params(self, non_default_params, optional_params, model, drop_params):
return optional_params
def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None):
return "https://images.example/v1/generations"
def transform_image_generation_request(self, model, prompt, optional_params, litellm_params, headers):
return {"prompt": prompt}
def transform_image_generation_response(
self,
model,
raw_response,
model_response,
logging_obj,
request_data,
optional_params,
litellm_params,
encoding,
api_key=None,
json_mode=None,
):
return ImageResponse(data=[ImageObject(b64_json=raw_response.json()["b64_json"])])
def _image_generation_call_kwargs() -> dict:
return {
"model": "image-model",
"prompt": "a cat",
"image_generation_provider_config": _HeaderImageGenerationConfig(),
"image_generation_optional_request_params": {},
"custom_llm_provider": "openai",
"litellm_params": {},
"logging_obj": Mock(),
"timeout": 10.0,
}
def test_image_generation_handler_records_upstream_response_headers():
client = HTTPHandler()
client.client = httpx.Client(transport=_json_with_upstream_headers({"b64_json": "abc"}))
response = BaseLLMHTTPHandler().image_generation_handler(client=client, **_image_generation_call_kwargs())
assert response.data[0].b64_json == "abc"
_assert_upstream_headers_recorded(response)
@pytest.mark.asyncio
async def test_async_image_generation_handler_records_upstream_response_headers():
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=_json_with_upstream_headers({"b64_json": "abc"}))
response = await BaseLLMHTTPHandler().async_image_generation_handler(
client=client, **_image_generation_call_kwargs()
)
assert response.data[0].b64_json == "abc"
_assert_upstream_headers_recorded(response)
class _HeaderTextToSpeechConfig(BaseTextToSpeechConfig):
def get_supported_openai_params(self, model):
return []
def map_openai_params(self, model, optional_params, voice=None, drop_params=False, kwargs=None):
return voice, optional_params
def validate_environment(self, headers, model, api_key=None, api_base=None):
return {}
def get_complete_url(self, model, api_base, litellm_params):
return "https://tts.example/v1/speech"
def transform_text_to_speech_request(self, model, input, voice, optional_params, litellm_params, headers):
return {"dict_body": {"input": input}}
def transform_text_to_speech_response(self, model, raw_response, logging_obj):
return HttpxBinaryResponseContent(response=raw_response)
def _text_to_speech_call_kwargs() -> dict:
return {
"model": "tts-model",
"input": "hello",
"voice": "alloy",
"text_to_speech_provider_config": _HeaderTextToSpeechConfig(),
"text_to_speech_optional_params": {},
"custom_llm_provider": "openai",
"litellm_params": {},
"logging_obj": Mock(),
"timeout": 10.0,
}
def test_text_to_speech_handler_records_upstream_response_headers():
client = HTTPHandler()
client.client = httpx.Client(transport=_binary_with_upstream_headers())
response = BaseLLMHTTPHandler().text_to_speech_handler(client=client, **_text_to_speech_call_kwargs())
assert response.content == b"audio-bytes"
_assert_upstream_headers_recorded(response)
@pytest.mark.asyncio
async def test_async_text_to_speech_handler_records_upstream_response_headers():
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=_binary_with_upstream_headers())
response = await BaseLLMHTTPHandler().async_text_to_speech_handler(client=client, **_text_to_speech_call_kwargs())
assert response.content == b"audio-bytes"
_assert_upstream_headers_recorded(response)
async def _get_by_id_with_upstream(handler_name: str, upstream_response: httpx.Response) -> object:
async_client: Final = AsyncHTTPHandler()
await async_client.close()
async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: upstream_response))
handler: Final = BaseLLMHTTPHandler()
if handler_name == "get_eval":
return await handler.async_get_eval_handler(
url="https://api.example.test/v1/evals/eval_missing",
evals_api_provider_config=OpenAIEvalsConfig(),
custom_llm_provider="openai",
litellm_params=GenericLiteLLMParams(),
logging_obj=Mock(),
client=async_client,
)
return await handler.async_get_skill_handler(
url="https://api.example.test/v1/skills/skill_missing",
skills_api_provider_config=AnthropicSkillsConfig(),
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(),
logging_obj=Mock(),
client=async_client,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("handler_name", ("get_eval", "get_skill"))
@pytest.mark.parametrize("status_code", (400, 401, 404, 429, 503))
async def test_get_by_id_handlers_raise_the_provider_error_status(handler_name: str, status_code: int) -> None:
upstream_response: Final = httpx.Response(status_code, json={"error": {"message": "No such object"}})
with pytest.raises(BaseLLMException) as error:
await _get_by_id_with_upstream(handler_name, upstream_response)
assert error.value.status_code == status_code
assert "No such object" in error.value.message
def _clients_answering_with(upstream_response: httpx.Response) -> tuple[HTTPHandler, AsyncHTTPHandler]:
sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(lambda _: upstream_response)))
async_client: Final = AsyncHTTPHandler()
async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _: upstream_response))
return sync_client, async_client
def _call_lookup_handler(name: str, is_async: bool, client: HTTPHandler | AsyncHTTPHandler) -> object:
handler: Final = BaseLLMHTTPHandler()
vector_store_params: Final = GenericLiteLLMParams(api_base="https://api.example.test/v1", api_key="sk-test")
files_params: Final = {"api_base": "https://api.example.test", "api_key": "sk-test"}
match name:
case "vector_store_retrieve":
return handler.vector_store_retrieve_handler(
vector_store_id="vs_missing",
vector_store_provider_config=OpenAIVectorStoreConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "vector_store_list":
return handler.vector_store_list_handler(
after=None,
before=None,
limit=None,
order=None,
vector_store_provider_config=OpenAIVectorStoreConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "vector_store_file_list":
return handler.vector_store_file_list_handler(
vector_store_id="vs_missing",
query_params={},
vector_store_files_provider_config=OpenAIVectorStoreFilesConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "vector_store_file_retrieve":
return handler.vector_store_file_retrieve_handler(
vector_store_id="vs_missing",
file_id="file_missing",
vector_store_files_provider_config=OpenAIVectorStoreFilesConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "file_retrieve":
return handler.retrieve_file(
file_id="file_missing",
provider_config=MistralFilesConfig(),
litellm_params=files_params,
headers={},
logging_obj=Mock(),
_is_async=is_async,
client=client,
)
case "vector_store_file_content":
return handler.vector_store_file_content_handler(
vector_store_id="vs_missing",
file_id="file_missing",
vector_store_files_provider_config=OpenAIVectorStoreFilesConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "eval_list":
return handler.list_evals_handler(
url="https://api.example.test/v1/evals",
query_params={},
evals_api_provider_config=OpenAIEvalsConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "eval_get":
return handler.get_eval_handler(
url="https://api.example.test/v1/evals/eval_missing",
evals_api_provider_config=OpenAIEvalsConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "eval_run_list":
return handler.list_runs_handler(
url="https://api.example.test/v1/evals/eval_missing/runs",
query_params={},
evals_api_provider_config=OpenAIEvalsConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "eval_run_get":
return handler.get_run_handler(
url="https://api.example.test/v1/evals/eval_missing/runs/run_missing",
evals_api_provider_config=OpenAIEvalsConfig(),
custom_llm_provider="openai",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "skill_list":
return handler.list_skills_handler(
url="https://api.example.test/v1/skills",
query_params={},
skills_api_provider_config=AnthropicSkillsConfig(),
custom_llm_provider="anthropic",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case "skill_get":
return handler.get_skill_handler(
url="https://api.example.test/v1/skills/skill_missing",
skills_api_provider_config=AnthropicSkillsConfig(),
custom_llm_provider="anthropic",
litellm_params=vector_store_params,
logging_obj=Mock(),
client=client,
_is_async=is_async,
)
case _:
return handler.list_files(
purpose=None,
provider_config=MistralFilesConfig(),
litellm_params=files_params,
headers={},
logging_obj=Mock(),
_is_async=is_async,
client=client,
)
async def _run_lookup_handler(name: str, is_async: bool, client: HTTPHandler | AsyncHTTPHandler) -> object:
result: Final = _call_lookup_handler(name, is_async, client)
return await result if inspect.isawaitable(result) else result
@pytest.mark.asyncio
@pytest.mark.parametrize(
"name",
(
"vector_store_retrieve",
"vector_store_list",
"vector_store_file_list",
"vector_store_file_retrieve",
"vector_store_file_content",
"file_retrieve",
"file_list",
"eval_list",
"eval_get",
"eval_run_list",
"eval_run_get",
"skill_list",
"skill_get",
),
)
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.parametrize("status_code", (404, 503))
async def test_lookup_handlers_raise_the_provider_error_status(name: str, is_async: bool, status_code: int) -> None:
sync_client, async_client = _clients_answering_with(
httpx.Response(status_code, json={"error": {"message": "No such object"}})
)
with pytest.raises(BaseLLMException) as error:
await _run_lookup_handler(name, is_async, async_client if is_async else sync_client)
assert error.value.status_code == status_code
assert "No such object" in error.value.message
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.parametrize(
("status_code", "content", "headers", "expected_bytes"),
(
(
416,
b"<Error><Code>InvalidRange</Code><ActualObjectSize>0</ActualObjectSize></Error>",
{},
0,
),
(206, b"", {"Content-Range": "bytes 0-0/4321"}, 4321),
),
)
async def test_retrieve_file_accepts_bedrock_successful_range_responses(
is_async: bool,
status_code: int,
content: bytes,
headers: dict[str, str],
expected_bytes: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket")
file_id = "s3://my-bucket/litellm-batch-outputs/job-123/output.jsonl"
transport = httpx.MockTransport(
lambda request: httpx.Response(
status_code,
content=content,
headers=headers,
request=request,
)
)
params = {
"aws_access_key_id": "AKIAEXAMPLE",
"aws_secret_access_key": "secret",
"aws_region_name": "us-west-2",
}
handler = BaseLLMHTTPHandler()
if is_async:
client = AsyncHTTPHandler()
await client.close()
async_client = httpx.AsyncClient(transport=transport)
client.client = async_client
try:
result = await handler.async_retrieve_file(
file_id=file_id,
provider_config=BedrockFilesConfig(),
litellm_params=params,
headers={},
logging_obj=Mock(),
client=client,
)
finally:
await async_client.aclose()
else:
sync_client = httpx.Client(transport=transport)
client = HTTPHandler(client=sync_client)
try:
result = handler.retrieve_file(
file_id=file_id,
provider_config=BedrockFilesConfig(),
litellm_params=params,
headers={},
logging_obj=Mock(),
client=client,
)
finally:
sync_client.close()
assert result.bytes == expected_bytes
assert result.filename == "output.jsonl"
assert result.purpose == "batch_output"
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", (False, True))
async def test_retrieve_file_rejects_unverified_bedrock_range_error(
is_async: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket")
file_id = "s3://my-bucket/litellm-batch-outputs/job-123/output.jsonl"
transport = httpx.MockTransport(
lambda request: httpx.Response(
416,
content=b"<Error><Code>InvalidRange</Code></Error>",
request=request,
)
)
params = {
"aws_access_key_id": "AKIAEXAMPLE",
"aws_secret_access_key": "secret",
"aws_region_name": "us-west-2",
}
handler = BaseLLMHTTPHandler()
if is_async:
client = AsyncHTTPHandler()
await client.close()
async_client = httpx.AsyncClient(transport=transport)
client.client = async_client
try:
with pytest.raises(BaseLLMException, match="InvalidRange"):
await handler.async_retrieve_file(
file_id=file_id,
provider_config=BedrockFilesConfig(),
litellm_params=params,
headers={},
logging_obj=Mock(),
client=client,
)
finally:
await async_client.aclose()
else:
sync_client = httpx.Client(transport=transport)
client = HTTPHandler(client=sync_client)
try:
with pytest.raises(BaseLLMException, match="InvalidRange"):
handler.retrieve_file(
file_id=file_id,
provider_config=BedrockFilesConfig(),
litellm_params=params,
headers={},
logging_obj=Mock(),
client=client,
)
finally:
sync_client.close()
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_its_beta() -> None:
received: Final[list[httpx.Request]] = []
def anthropic_answers(request: httpx.Request) -> httpx.Response:
received.append(request)
return httpx.Response(
status_code=200,
json={
"id": "msg_dropped_safeguards",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 5, "output_tokens": 1},
},
request=request,
)
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(anthropic_answers))
try:
await litellm.anthropic.messages.acreate(
max_tokens=16,
messages=[{"role": "user", "content": "hi"}],
model="anthropic/claude-sonnet-4-6",
api_key="sk-ant-test",
safeguards=({"type": "dangerous_tool_use"},),
additional_drop_params=["safeguards"],
client=upstream,
)
finally:
await upstream.close()
assert len(received) == 1, received
sent: Final = received[0]
assert "safeguards" not in json.loads(sent.content)
assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers)