Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Regression tests for transport errors returned by the LiteLLM backend."""
from __future__ import annotations
import json
from collections.abc import AsyncIterator
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from tests._dotenv import importorskip_no_env_leak
importorskip_no_env_leak("litellm")
from fastapi.testclient import TestClient # noqa: E402
from headroom.backends.anyllm import AnyLLMBackend # noqa: E402
from headroom.backends.base import BackendResponse # noqa: E402
from headroom.backends.litellm import LiteLLMBackend # noqa: E402
from headroom.proxy.public_errors import ( # noqa: E402
UPSTREAM_PROTOCOL_ERROR,
UPSTREAM_TIMEOUT,
public_message,
)
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
# Transport exceptions never surface their own text (it names the upstream host);
# an empty-message ReadError/ReadTimeout maps to the fixed vocabulary instead.
_PROTOCOL = public_message(UPSTREAM_PROTOCOL_ERROR)
_TIMEOUT = public_message(UPSTREAM_TIMEOUT)
_BODY = {
"model": "claude-sonnet-4-20250514",
"messages": [{"role": "user", "content": "hello"}],
}
@pytest.mark.asyncio
async def test_send_message_names_transport_error_without_message() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadError(""),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_message(_BODY, {})
assert result.status_code == 500
assert result.error == _PROTOCOL
assert result.body["error"]["message"] == _PROTOCOL
@pytest.mark.asyncio
async def test_stream_message_names_transport_error_without_message() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
events = [event async for event in backend.stream_message(_BODY, {})]
error_event = next(event for event in events if event.event_type == "error")
assert error_event.data["error"]["message"] == _TIMEOUT
@pytest.mark.asyncio
async def test_anyllm_backend_names_transport_error_without_message() -> None:
pytest.importorskip("any_llm")
fake_llm = MagicMock()
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
backend = AnyLLMBackend(provider="anthropic")
result = await backend.send_message(_BODY, {})
assert result.status_code == 500
assert result.error == _PROTOCOL
assert result.body["error"]["message"] == _PROTOCOL
@pytest.mark.asyncio
async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:
pytest.importorskip("any_llm")
fake_llm = MagicMock()
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
backend = AnyLLMBackend(provider="anthropic")
events = [event async for event in backend.stream_message(_BODY, {})]
error_event = next(event for event in events if event.event_type == "error")
assert error_event.data["error"]["message"] == _TIMEOUT
@pytest.mark.asyncio
async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:
pytest.importorskip("any_llm")
with (
patch(
"headroom.backends.anyllm.AnyLLM.create",
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
),
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
):
anyllm_backend = AnyLLMBackend(provider="openai")
anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})
litellm_backend = LiteLLMBackend(provider="openrouter")
litellm_result = await litellm_backend.send_openai_message(_BODY, {})
assert anyllm_result.body["error"]["message"] == _PROTOCOL
assert anyllm_result.error == _PROTOCOL
assert litellm_result.body["error"]["message"] == _TIMEOUT
assert litellm_result.error == _TIMEOUT
@pytest.mark.asyncio
async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:
pytest.importorskip("any_llm")
with (
patch(
"headroom.backends.anyllm.AnyLLM.create",
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
),
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
):
anyllm_backend = AnyLLMBackend(provider="openai")
anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]
litellm_backend = LiteLLMBackend(provider="openrouter")
litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]
assert f'"message": {json.dumps(_PROTOCOL)}' in anyllm_chunks[0]
assert f'"message": {json.dumps(_TIMEOUT)}' in litellm_chunks[0]
def _erroring_anthropic_backend() -> MagicMock:
"""Raise blank-message transport errors through both proxy backend paths."""
async def send_message(body: dict, headers: dict) -> BackendResponse:
raise httpx.ReadError("")
async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:
raise httpx.ReadTimeout("")
yield # pragma: no cover - keeps this function an async generator
backend = MagicMock()
backend.name = "anyllm-anthropic"
backend.send_message = send_message
backend.stream_message = stream_message
backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
backend.supports_model = MagicMock(return_value=True)
return backend
def _proxy_config() -> ProxyConfig:
return ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
backend="anyllm",
anyllm_provider="anthropic",
)
def _messages_request(*, stream: bool) -> dict:
return {
"model": "claude-3-5-sonnet-20241022",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 32,
"stream": stream,
}
def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:
backend = _erroring_anthropic_backend()
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
app = create_app(_proxy_config())
with TestClient(app) as client:
response = client.post(
"/v1/messages",
json=_messages_request(stream=False),
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
)
assert response.status_code == 500
# Proxy-level replies append the request id for log correlation.
assert response.json()["error"]["message"].startswith(_PROTOCOL)
assert response.json()["error"]["code"] == UPSTREAM_PROTOCOL_ERROR
def test_bedrock_stream_names_transport_error_without_message() -> None:
backend = _erroring_anthropic_backend()
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
app = create_app(_proxy_config())
with TestClient(app) as client:
response = client.post(
"/v1/messages",
json=_messages_request(stream=True),
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
)
assert response.status_code == 200
assert _TIMEOUT in response.text
assert '"code": "upstream_timeout"' in response.text
def _upstream_error(status_code: int, message: str) -> Exception:
"""A LiteLLM-style exception: carries the upstream HTTP status as an attribute."""
import litellm
classes = {
400: litellm.BadRequestError,
401: litellm.AuthenticationError,
403: litellm.PermissionDeniedError,
404: litellm.NotFoundError,
422: litellm.UnprocessableEntityError,
429: litellm.RateLimitError,
500: litellm.InternalServerError,
}
if status_code not in classes:
# Unmapped 4xx: litellm bakes the status into each class, so use a bare
# exception carrying only the ``status_code`` attribute.
err = Exception(message)
err.status_code = status_code # type: ignore[attr-defined]
return err
response = httpx.Response(status_code, request=httpx.Request("POST", "https://upstream.test"))
return classes[status_code](
message=message, model="bedrock/claude", llm_provider="bedrock", response=response
)
_BAD_THINKING = (
"BedrockException - thinking.adaptive.display: Input should be 'summarized', 'omitted'"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status", "error_type"),
[
(400, "invalid_request_error"),
(403, "permission_error"),
(422, "invalid_request_error"),
(401, "authentication_error"),
(404, "not_found_error"),
(429, "rate_limit_error"),
(402, "invalid_request_error"),
(408, "invalid_request_error"),
(409, "invalid_request_error"),
(500, "api_error"),
],
)
async def test_send_message_keeps_upstream_litellm_status(status: int, error_type: str) -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=_upstream_error(status, _BAD_THINKING),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_message(_BODY, {})
assert result.status_code == status
assert result.body["error"]["type"] == error_type
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status", "error_type"),
[
(400, "invalid_request_error"),
(403, "permission_error"),
(422, "invalid_request_error"),
(401, "invalid_api_key"),
(404, "model_not_found"),
(429, "rate_limit_exceeded"),
(402, "invalid_request_error"),
(408, "invalid_request_error"),
(409, "invalid_request_error"),
(500, "api_error"),
],
)
async def test_send_openai_message_keeps_upstream_litellm_status(
status: int, error_type: str
) -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=_upstream_error(status, _BAD_THINKING),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_openai_message(_BODY, {})
assert result.status_code == status
assert result.body["error"]["type"] == error_type
@pytest.mark.asyncio
async def test_send_message_substring_fallback_without_status_attribute() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=RuntimeError("Authentication failed: bad credentials"),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_message(_BODY, {})
assert result.status_code == 401
assert result.body["error"]["type"] == "authentication_error"
class _StatusError(Exception):
def __init__(self, status_code: object) -> None:
super().__init__("payload too large")
self.status_code = status_code
@pytest.mark.asyncio
@pytest.mark.parametrize(
("method", "error_type"),
[("send_message", "request_too_large"), ("send_openai_message", "invalid_request_error")],
)
async def test_413_status_attribute_is_kept(method: str, error_type: str) -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=_StatusError(413),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await getattr(backend, method)(_BODY, {})
assert result.status_code == 413
assert result.body["error"]["type"] == error_type
@pytest.mark.asyncio
async def test_non_int_status_attribute_falls_back_to_substring() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=_StatusError("429"),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_message(_BODY, {})
assert result.status_code == 500
assert result.body["error"]["type"] == "api_error"