Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): preserve empty transport error context (#3675)

AxelRayca02f28c6744
diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.py
index c1e02b35f..b7a089bf7 100644
--- a/headroom/backends/anyllm.py
+++ b/headroom/backends/anyllm.py
@@ -12,6 +12,8 @@ import uuid
from collections.abc import AsyncIterator
from typing import Any, cast
+from headroom.utils import format_exception_message
+
from .base import Backend, BackendResponse, StreamEvent
logger = logging.getLogger(__name__)
@@ -305,7 +307,7 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"any-llm error: {e}")
+ logger.error(f"any-llm error: {format_exception_message(e)}")
return self._error_response(e)
async def stream_message(
@@ -497,12 +499,13 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"any-llm streaming error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"any-llm streaming error: {error_message}")
yield StreamEvent(
event_type="error",
data={
"type": "error",
- "error": {"type": "api_error", "message": str(e)},
+ "error": {"type": "api_error", "message": error_message},
},
)
@@ -590,7 +593,7 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"any-llm OpenAI error: {e}")
+ logger.error(f"any-llm OpenAI error: {format_exception_message(e)}")
return self._error_response(e, openai_format=True)
def _error_response(self, e: Exception, openai_format: bool = False) -> BackendResponse:
@@ -598,6 +601,7 @@ class AnyLLMBackend(Backend):
error_type = "api_error"
status_code = 500
+ error_message = format_exception_message(e)
error_str = str(e).lower()
if "authentication" in error_str or "api_key" in error_str or "api key" in error_str:
error_type = "invalid_api_key" if openai_format else "authentication_error"
@@ -611,11 +615,11 @@ class AnyLLMBackend(Backend):
body: dict[str, Any]
if openai_format:
- body = {"error": {"message": str(e), "type": error_type, "code": error_type}}
+ body = {"error": {"message": error_message, "type": error_type, "code": error_type}}
else:
- body = {"type": "error", "error": {"type": error_type, "message": str(e)}}
+ body = {"type": "error", "error": {"type": error_type, "message": error_message}}
- return BackendResponse(body=body, status_code=status_code, error=str(e))
+ return BackendResponse(body=body, status_code=status_code, error=error_message)
async def stream_openai_message(
self,
@@ -661,10 +665,11 @@ class AnyLLMBackend(Backend):
yield "data: [DONE]\n\n"
except Exception as e:
- logger.error(f"any-llm OpenAI streaming error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"any-llm OpenAI streaming error: {error_message}")
error_data = {
"error": {
- "message": str(e),
+ "message": error_message,
"type": "api_error",
"code": "backend_error",
}
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 2bf5d0d79..805cf6df8 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -21,6 +21,8 @@ from collections.abc import AsyncIterator
from dataclasses import dataclass, field
from typing import Any
+from headroom.utils import format_exception_message
+
from .base import Backend, BackendResponse, StreamEvent
logger = logging.getLogger(__name__)
@@ -1261,7 +1263,8 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"LiteLLM error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"LiteLLM error: {error_message}")
# Map to Anthropic error format
error_type = "api_error"
@@ -1281,10 +1284,10 @@ class LiteLLMBackend(Backend):
return BackendResponse(
body={
"type": "error",
- "error": {"type": error_type, "message": str(e)},
+ "error": {"type": error_type, "message": error_message},
},
status_code=status_code,
- error=str(e),
+ error=error_message,
)
async def stream_message(
@@ -1671,12 +1674,13 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"LiteLLM streaming error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"LiteLLM streaming error: {error_message}")
yield StreamEvent(
event_type="error",
data={
"type": "error",
- "error": {"type": "api_error", "message": str(e)},
+ "error": {"type": "api_error", "message": error_message},
},
)
@@ -1871,7 +1875,8 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
- logger.error(f"LiteLLM OpenAI error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"LiteLLM OpenAI error: {error_message}")
# Map to OpenAI error format
error_type = "api_error"
@@ -1891,13 +1896,13 @@ class LiteLLMBackend(Backend):
return BackendResponse(
body={
"error": {
- "message": str(e),
+ "message": error_message,
"type": error_type,
"code": error_type,
}
},
status_code=status_code,
- error=str(e),
+ error=error_message,
)
async def stream_openai_message(
@@ -1970,10 +1975,11 @@ class LiteLLMBackend(Backend):
yield "data: [DONE]\n\n"
except Exception as e:
- logger.error(f"LiteLLM OpenAI streaming error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"LiteLLM OpenAI streaming error: {error_message}")
error_data = {
"error": {
- "message": str(e),
+ "message": error_message,
"type": "api_error",
"code": "backend_error",
}
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 492439992..41ec9b832 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -64,6 +64,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
from headroom.proxy.outcome import RequestOutcome
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
+from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
@@ -3694,12 +3695,15 @@ class AnthropicHandlerMixin:
content=backend_response.body,
)
except Exception as e:
- logger.error(f"[{request_id}] Bedrock backend error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"[{request_id}] Bedrock backend error: {error_message}")
+ # Unit 4: release the pre-upstream semaphore on error.
+ await _finalize_pre_upstream()
return JSONResponse(
status_code=500,
content={
"type": "error",
- "error": {"type": "api_error", "message": str(e)},
+ "error": {"type": "api_error", "message": error_message},
},
)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index a6fc34ff5..3c48927b9 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -30,6 +30,7 @@ import httpx
from headroom.copilot_auth import apply_copilot_api_auth
from headroom.proxy.stream_output_tokens import estimate_output_tokens
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
+from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
@@ -2018,10 +2019,11 @@ class StreamingMixin:
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
except Exception as e:
- logger.error(f"[{request_id}] Bedrock streaming error: {e}")
+ error_message = format_exception_message(e)
+ logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
error_event = {
"type": "error",
- "error": {"type": "api_error", "message": str(e)},
+ "error": {"type": "api_error", "message": error_message},
}
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
diff --git a/headroom/utils.py b/headroom/utils.py
index 4a559e48d..a0deb5c80 100644
--- a/headroom/utils.py
+++ b/headroom/utils.py
@@ -20,6 +20,12 @@ def generate_request_id() -> str:
return str(uuid.uuid4())
+def format_exception_message(exc: BaseException) -> str:
+ """Return exception text with a useful fallback for empty messages."""
+ message = str(exc)
+ return message or f"{type(exc).__name__} (no message)"
+
+
def compute_hash(data: str | bytes) -> str:
"""Compute SHA256 hash, returning hex string."""
if isinstance(data, str):
diff --git a/tests/test_litellm_transport_errors.py b/tests/test_litellm_transport_errors.py
new file mode 100644
index 000000000..735f41dac
--- /dev/null
+++ b/tests/test_litellm_transport_errors.py
@@ -0,0 +1,205 @@
+"""Regression tests for transport errors returned by the LiteLLM backend."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import pytest
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.backends.anyllm import AnyLLMBackend # noqa: E402
+from headroom.backends.base import BackendResponse # noqa: E402
+from headroom.backends.litellm import LiteLLMBackend # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+_BODY = {
+ "model": "claude-sonnet-4-20250514",
+ "messages": [{"role": "user", "content": "hello"}],
+}
+
+
+async def test_send_message_names_transport_error_without_message() -> None:
+ with (
+ patch(
+ "headroom.backends.litellm.acompletion",
+ new_callable=AsyncMock,
+ side_effect=httpx.ReadError(""),
+ ),
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
+ result = await backend.send_message(_BODY, {})
+
+ assert result.status_code == 500
+ assert result.error == "ReadError (no message)"
+ assert result.body["error"]["message"] == "ReadError (no message)"
+
+
+async def test_stream_message_names_transport_error_without_message() -> None:
+ with (
+ patch(
+ "headroom.backends.litellm.acompletion",
+ new_callable=AsyncMock,
+ side_effect=httpx.ReadTimeout(""),
+ ),
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
+ events = [event async for event in backend.stream_message(_BODY, {})]
+
+ error_event = next(event for event in events if event.event_type == "error")
+ assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
+
+
+async def test_anyllm_backend_names_transport_error_without_message() -> None:
+ pytest.importorskip("any_llm")
+ fake_llm = MagicMock()
+ fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))
+
+ with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
+ backend = AnyLLMBackend(provider="anthropic")
+ result = await backend.send_message(_BODY, {})
+
+ assert result.status_code == 500
+ assert result.error == "ReadError (no message)"
+ assert result.body["error"]["message"] == "ReadError (no message)"
+
+
+async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:
+ pytest.importorskip("any_llm")
+ fake_llm = MagicMock()
+ fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))
+
+ with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
+ backend = AnyLLMBackend(provider="anthropic")
+ events = [event async for event in backend.stream_message(_BODY, {})]
+
+ error_event = next(event for event in events if event.event_type == "error")
+ assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
+
+
+async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:
+ pytest.importorskip("any_llm")
+ with (
+ patch(
+ "headroom.backends.anyllm.AnyLLM.create",
+ return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
+ ),
+ patch(
+ "headroom.backends.litellm.acompletion",
+ new_callable=AsyncMock,
+ side_effect=httpx.ReadTimeout(""),
+ ),
+ ):
+ anyllm_backend = AnyLLMBackend(provider="openai")
+ anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})
+ litellm_backend = LiteLLMBackend(provider="openrouter")
+ litellm_result = await litellm_backend.send_openai_message(_BODY, {})
+
+ assert anyllm_result.body["error"]["message"] == "ReadError (no message)"
+ assert anyllm_result.error == "ReadError (no message)"
+ assert litellm_result.body["error"]["message"] == "ReadTimeout (no message)"
+ assert litellm_result.error == "ReadTimeout (no message)"
+
+
+async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:
+ pytest.importorskip("any_llm")
+ with (
+ patch(
+ "headroom.backends.anyllm.AnyLLM.create",
+ return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
+ ),
+ patch(
+ "headroom.backends.litellm.acompletion",
+ new_callable=AsyncMock,
+ side_effect=httpx.ReadTimeout(""),
+ ),
+ ):
+ anyllm_backend = AnyLLMBackend(provider="openai")
+ anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]
+ litellm_backend = LiteLLMBackend(provider="openrouter")
+ litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]
+
+ assert '"message": "ReadError (no message)"' in anyllm_chunks[0]
+ assert '"message": "ReadTimeout (no message)"' in litellm_chunks[0]
+
+
+def _erroring_anthropic_backend() -> MagicMock:
+ """Raise blank-message transport errors through both proxy backend paths."""
+
+ async def send_message(body: dict, headers: dict) -> BackendResponse:
+ raise httpx.ReadError("")
+
+ async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:
+ raise httpx.ReadTimeout("")
+ yield # pragma: no cover - keeps this function an async generator
+
+ backend = MagicMock()
+ backend.name = "anyllm-anthropic"
+ backend.send_message = send_message
+ backend.stream_message = stream_message
+ backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
+ backend.supports_model = MagicMock(return_value=True)
+ return backend
+
+
+def _proxy_config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="anthropic",
+ )
+
+
+def _messages_request(*, stream: bool) -> dict:
+ return {
+ "model": "claude-3-5-sonnet-20241022",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 32,
+ "stream": stream,
+ }
+
+
+def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:
+ backend = _erroring_anthropic_backend()
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(_proxy_config())
+ with TestClient(app) as client:
+ response = client.post(
+ "/v1/messages",
+ json=_messages_request(stream=False),
+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
+ )
+
+ assert response.status_code == 500
+ assert response.json()["error"]["message"] == "ReadError (no message)"
+
+
+def test_bedrock_stream_names_transport_error_without_message() -> None:
+ backend = _erroring_anthropic_backend()
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(_proxy_config())
+ with TestClient(app) as client:
+ response = client.post(
+ "/v1/messages",
+ json=_messages_request(stream=True),
+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
+ )
+
+ assert response.status_code == 200
+ assert '"message": "ReadTimeout (no message)"' in response.text