Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): preserve empty transport error context (#3675)
AxelRayca02f28c6744
diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.pyindex c1e02b35f..b7a089bf7 100644--- a/headroom/backends/anyllm.py+++ b/headroom/backends/anyllm.py@@ -12,6 +12,8 @@ import uuidfrom collections.abc import AsyncIteratorfrom typing import Any, cast+from headroom.utils import format_exception_message+from .base import Backend, BackendResponse, StreamEventlogger = logging.getLogger(__name__)@@ -305,7 +307,7 @@ class AnyLLMBackend(Backend):)except Exception as e:- logger.error(f"any-llm error: {e}")+ logger.error(f"any-llm error: {format_exception_message(e)}")return self._error_response(e)async def stream_message(@@ -497,12 +499,13 @@ class AnyLLMBackend(Backend):)except Exception as e:- logger.error(f"any-llm streaming error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"any-llm streaming error: {error_message}")yield StreamEvent(event_type="error",data={"type": "error",- "error": {"type": "api_error", "message": str(e)},+ "error": {"type": "api_error", "message": error_message},},)@@ -590,7 +593,7 @@ class AnyLLMBackend(Backend):)except Exception as e:- logger.error(f"any-llm OpenAI error: {e}")+ logger.error(f"any-llm OpenAI error: {format_exception_message(e)}")return self._error_response(e, openai_format=True)def _error_response(self, e: Exception, openai_format: bool = False) -> BackendResponse:@@ -598,6 +601,7 @@ class AnyLLMBackend(Backend):error_type = "api_error"status_code = 500+ error_message = format_exception_message(e)error_str = str(e).lower()if "authentication" in error_str or "api_key" in error_str or "api key" in error_str:error_type = "invalid_api_key" if openai_format else "authentication_error"@@ -611,11 +615,11 @@ class AnyLLMBackend(Backend):body: dict[str, Any]if openai_format:- body = {"error": {"message": str(e), "type": error_type, "code": error_type}}+ body = {"error": {"message": error_message, "type": error_type, "code": error_type}}else:- body = {"type": "error", "error": {"type": error_type, "message": str(e)}}+ body = {"type": "error", "error": {"type": error_type, "message": error_message}}- return BackendResponse(body=body, status_code=status_code, error=str(e))+ return BackendResponse(body=body, status_code=status_code, error=error_message)async def stream_openai_message(self,@@ -661,10 +665,11 @@ class AnyLLMBackend(Backend):yield "data: [DONE]\n\n"except Exception as e:- logger.error(f"any-llm OpenAI streaming error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"any-llm OpenAI streaming error: {error_message}")error_data = {"error": {- "message": str(e),+ "message": error_message,"type": "api_error","code": "backend_error",}diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 2bf5d0d79..805cf6df8 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -21,6 +21,8 @@ from collections.abc import AsyncIteratorfrom dataclasses import dataclass, fieldfrom typing import Any+from headroom.utils import format_exception_message+from .base import Backend, BackendResponse, StreamEventlogger = logging.getLogger(__name__)@@ -1261,7 +1263,8 @@ class LiteLLMBackend(Backend):)except Exception as e:- logger.error(f"LiteLLM error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"LiteLLM error: {error_message}")# Map to Anthropic error formaterror_type = "api_error"@@ -1281,10 +1284,10 @@ class LiteLLMBackend(Backend):return BackendResponse(body={"type": "error",- "error": {"type": error_type, "message": str(e)},+ "error": {"type": error_type, "message": error_message},},status_code=status_code,- error=str(e),+ error=error_message,)async def stream_message(@@ -1671,12 +1674,13 @@ class LiteLLMBackend(Backend):)except Exception as e:- logger.error(f"LiteLLM streaming error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"LiteLLM streaming error: {error_message}")yield StreamEvent(event_type="error",data={"type": "error",- "error": {"type": "api_error", "message": str(e)},+ "error": {"type": "api_error", "message": error_message},},)@@ -1871,7 +1875,8 @@ class LiteLLMBackend(Backend):)except Exception as e:- logger.error(f"LiteLLM OpenAI error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"LiteLLM OpenAI error: {error_message}")# Map to OpenAI error formaterror_type = "api_error"@@ -1891,13 +1896,13 @@ class LiteLLMBackend(Backend):return BackendResponse(body={"error": {- "message": str(e),+ "message": error_message,"type": error_type,"code": error_type,}},status_code=status_code,- error=str(e),+ error=error_message,)async def stream_openai_message(@@ -1970,10 +1975,11 @@ class LiteLLMBackend(Backend):yield "data: [DONE]\n\n"except Exception as e:- logger.error(f"LiteLLM OpenAI streaming error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"LiteLLM OpenAI streaming error: {error_message}")error_data = {"error": {- "message": str(e),+ "message": error_message,"type": "api_error","code": "backend_error",}diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 492439992..41ec9b832 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -64,6 +64,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_replyfrom headroom.proxy.outcome import RequestOutcomefrom headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_forfrom headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens+from headroom.utils import format_exception_messagelogger = logging.getLogger("headroom.proxy")@@ -3694,12 +3695,15 @@ class AnthropicHandlerMixin:content=backend_response.body,)except Exception as e:- logger.error(f"[{request_id}] Bedrock backend error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"[{request_id}] Bedrock backend error: {error_message}")+ # Unit 4: release the pre-upstream semaphore on error.+ await _finalize_pre_upstream()return JSONResponse(status_code=500,content={"type": "error",- "error": {"type": "api_error", "message": str(e)},+ "error": {"type": "api_error", "message": error_message},},)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex a6fc34ff5..3c48927b9 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -30,6 +30,7 @@ import httpxfrom headroom.copilot_auth import apply_copilot_api_authfrom headroom.proxy.stream_output_tokens import estimate_output_tokensfrom headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens+from headroom.utils import format_exception_messagelogger = logging.getLogger("headroom.proxy")@@ -2018,10 +2019,11 @@ class StreamingMixin:logger.error(f"[{request_id}] Bedrock stream error: {event.data}")except Exception as e:- logger.error(f"[{request_id}] Bedrock streaming error: {e}")+ error_message = format_exception_message(e)+ logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")error_event = {"type": "error",- "error": {"type": "api_error", "message": str(e)},+ "error": {"type": "api_error", "message": error_message},}yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()diff --git a/headroom/utils.py b/headroom/utils.pyindex 4a559e48d..a0deb5c80 100644--- a/headroom/utils.py+++ b/headroom/utils.py@@ -20,6 +20,12 @@ def generate_request_id() -> str:return str(uuid.uuid4())+def format_exception_message(exc: BaseException) -> str:+ """Return exception text with a useful fallback for empty messages."""+ message = str(exc)+ return message or f"{type(exc).__name__} (no message)"++def compute_hash(data: str | bytes) -> str:"""Compute SHA256 hash, returning hex string."""if isinstance(data, str):diff --git a/tests/test_litellm_transport_errors.py b/tests/test_litellm_transport_errors.pynew file mode 100644index 000000000..735f41dac--- /dev/null+++ b/tests/test_litellm_transport_errors.py@@ -0,0 +1,205 @@+"""Regression tests for transport errors returned by the LiteLLM backend."""++from __future__ import annotations++from collections.abc import AsyncIterator+from unittest.mock import AsyncMock, MagicMock, patch++import httpx+import pytest++from tests._dotenv import importorskip_no_env_leak++importorskip_no_env_leak("litellm")++from fastapi.testclient import TestClient # noqa: E402++from headroom.backends.anyllm import AnyLLMBackend # noqa: E402+from headroom.backends.base import BackendResponse # noqa: E402+from headroom.backends.litellm import LiteLLMBackend # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++_BODY = {+ "model": "claude-sonnet-4-20250514",+ "messages": [{"role": "user", "content": "hello"}],+}+++async def test_send_message_names_transport_error_without_message() -> None:+ with (+ patch(+ "headroom.backends.litellm.acompletion",+ new_callable=AsyncMock,+ side_effect=httpx.ReadError(""),+ ),+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),+ ):+ backend = LiteLLMBackend(provider="bedrock", region="us-east-1")+ result = await backend.send_message(_BODY, {})++ assert result.status_code == 500+ assert result.error == "ReadError (no message)"+ assert result.body["error"]["message"] == "ReadError (no message)"+++async def test_stream_message_names_transport_error_without_message() -> None:+ with (+ patch(+ "headroom.backends.litellm.acompletion",+ new_callable=AsyncMock,+ side_effect=httpx.ReadTimeout(""),+ ),+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),+ ):+ backend = LiteLLMBackend(provider="bedrock", region="us-east-1")+ events = [event async for event in backend.stream_message(_BODY, {})]++ error_event = next(event for event in events if event.event_type == "error")+ assert error_event.data["error"]["message"] == "ReadTimeout (no message)"+++async def test_anyllm_backend_names_transport_error_without_message() -> None:+ pytest.importorskip("any_llm")+ fake_llm = MagicMock()+ fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))++ with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):+ backend = AnyLLMBackend(provider="anthropic")+ result = await backend.send_message(_BODY, {})++ assert result.status_code == 500+ assert result.error == "ReadError (no message)"+ assert result.body["error"]["message"] == "ReadError (no message)"+++async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:+ pytest.importorskip("any_llm")+ fake_llm = MagicMock()+ fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))++ with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):+ backend = AnyLLMBackend(provider="anthropic")+ events = [event async for event in backend.stream_message(_BODY, {})]++ error_event = next(event for event in events if event.event_type == "error")+ assert error_event.data["error"]["message"] == "ReadTimeout (no message)"+++async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:+ pytest.importorskip("any_llm")+ with (+ patch(+ "headroom.backends.anyllm.AnyLLM.create",+ return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),+ ),+ patch(+ "headroom.backends.litellm.acompletion",+ new_callable=AsyncMock,+ side_effect=httpx.ReadTimeout(""),+ ),+ ):+ anyllm_backend = AnyLLMBackend(provider="openai")+ anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})+ litellm_backend = LiteLLMBackend(provider="openrouter")+ litellm_result = await litellm_backend.send_openai_message(_BODY, {})++ assert anyllm_result.body["error"]["message"] == "ReadError (no message)"+ assert anyllm_result.error == "ReadError (no message)"+ assert litellm_result.body["error"]["message"] == "ReadTimeout (no message)"+ assert litellm_result.error == "ReadTimeout (no message)"+++async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:+ pytest.importorskip("any_llm")+ with (+ patch(+ "headroom.backends.anyllm.AnyLLM.create",+ return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),+ ),+ patch(+ "headroom.backends.litellm.acompletion",+ new_callable=AsyncMock,+ side_effect=httpx.ReadTimeout(""),+ ),+ ):+ anyllm_backend = AnyLLMBackend(provider="openai")+ anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]+ litellm_backend = LiteLLMBackend(provider="openrouter")+ litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]++ assert '"message": "ReadError (no message)"' in anyllm_chunks[0]+ assert '"message": "ReadTimeout (no message)"' in litellm_chunks[0]+++def _erroring_anthropic_backend() -> MagicMock:+ """Raise blank-message transport errors through both proxy backend paths."""++ async def send_message(body: dict, headers: dict) -> BackendResponse:+ raise httpx.ReadError("")++ async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:+ raise httpx.ReadTimeout("")+ yield # pragma: no cover - keeps this function an async generator++ backend = MagicMock()+ backend.name = "anyllm-anthropic"+ backend.send_message = send_message+ backend.stream_message = stream_message+ backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")+ backend.supports_model = MagicMock(return_value=True)+ return backend+++def _proxy_config() -> ProxyConfig:+ return ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ backend="anyllm",+ anyllm_provider="anthropic",+ )+++def _messages_request(*, stream: bool) -> dict:+ return {+ "model": "claude-3-5-sonnet-20241022",+ "messages": [{"role": "user", "content": "hello"}],+ "max_tokens": 32,+ "stream": stream,+ }+++def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:+ backend = _erroring_anthropic_backend()+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):+ app = create_app(_proxy_config())+ with TestClient(app) as client:+ response = client.post(+ "/v1/messages",+ json=_messages_request(stream=False),+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},+ )++ assert response.status_code == 500+ assert response.json()["error"]["message"] == "ReadError (no message)"+++def test_bedrock_stream_names_transport_error_without_message() -> None:+ backend = _erroring_anthropic_backend()+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):+ app = create_app(_proxy_config())+ with TestClient(app) as client:+ response = client.post(+ "/v1/messages",+ json=_messages_request(stream=True),+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},+ )++ assert response.status_code == 200+ assert '"message": "ReadTimeout (no message)"' in response.text