Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): record streaming upstream 5xx as failed instead of success (#2682)

Omar Gerardo4949cd5594f4
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 7e8d49f7c..66e89fae3 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -931,6 +931,7 @@ class StreamingMixin:
parsed_response: dict[str, Any] | None = None,
client: str | None = None,
waste_signals: dict[str, int] | None = None,
+ status_code: int = 200,
# Set only by paths whose ``tokens_saved`` is the CONVERSATION's
# running total rather than this turn's -- OpenAI ``/v1/responses``,
# which re-sends and recompresses the whole transcript every turn.
@@ -1025,7 +1026,7 @@ class StreamingMixin:
# Prefix-tracker mutation is provider-specific state that lives
# outside the metric funnel. Run it before the funnel so the next
# request inherits correct prefix state regardless of metric path.
- if prefix_tracker is not None:
+ if 200 <= status_code < 300 and prefix_tracker is not None:
import copy as _copy
forwarded_messages = body.get("messages", [])
@@ -1108,6 +1109,7 @@ class StreamingMixin:
overhead_ms=optimization_latency,
tags=outcome_tags,
client=client,
+ status_code=status_code,
log_full_messages=getattr(self.config, "log_full_messages", False),
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
@@ -1532,6 +1534,7 @@ class StreamingMixin:
original_messages=original_messages,
client=client,
waste_signals=waste_signals,
+ status_code=upstream_response.status_code,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
)
@@ -1813,6 +1816,7 @@ class StreamingMixin:
parsed_response=parsed_response,
client=client,
waste_signals=waste_signals,
+ status_code=upstream_response.status_code,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
)
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 3f0897f56..545cf9f73 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -319,6 +319,7 @@ class RequestOutcome:
overhead_ms: float,
tags: dict[str, str] | None,
client: str | None,
+ status_code: int = 200,
log_full_messages: bool = False,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
@@ -411,6 +412,7 @@ class RequestOutcome:
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
attempted_input_tokens=optimized_tokens + tokens_saved,
+ status_code=status_code,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
cache_write_5m_tokens=cache_write_5m_tokens,
diff --git a/tests/test_outcome_records_5xx_as_failed.py b/tests/test_outcome_records_5xx_as_failed.py
index 5e6b95657..0c2203b31 100644
--- a/tests/test_outcome_records_5xx_as_failed.py
+++ b/tests/test_outcome_records_5xx_as_failed.py
@@ -57,3 +57,52 @@ def test_503_recorded_as_failed():
asyncio.run(emit_request_outcome(handler, _outcome(503)))
assert handler.metrics.failed == ["anthropic"]
assert handler.metrics.requested == []
+
+
+def test_stream_outcome_529_recorded_as_failed_and_skips_success_funnel():
+ handler = _Handler()
+ outcome = RequestOutcome.from_stream(
+ body={"messages": []},
+ provider="anthropic",
+ model="claude-opus-4-8",
+ request_id="req-stream-529",
+ original_tokens=100,
+ optimized_tokens=80,
+ output_tokens=0,
+ tokens_saved=20,
+ transforms_applied=[],
+ total_latency_ms=100.0,
+ overhead_ms=10.0,
+ tags=None,
+ client=None,
+ status_code=529,
+ )
+
+ assert outcome.status_code == 529
+ asyncio.run(emit_request_outcome(handler, outcome))
+ assert handler.metrics.failed == ["anthropic"]
+ assert handler.metrics.requested == []
+
+
+def test_stream_outcome_without_status_code_defaults_to_success():
+ handler = _Handler()
+ outcome = RequestOutcome.from_stream(
+ body={"messages": []},
+ provider="anthropic",
+ model="claude-opus-4-8",
+ request_id="req-stream-200",
+ original_tokens=100,
+ optimized_tokens=80,
+ output_tokens=10,
+ tokens_saved=20,
+ transforms_applied=[],
+ total_latency_ms=100.0,
+ overhead_ms=10.0,
+ tags=None,
+ client=None,
+ )
+
+ assert outcome.status_code == 200
+ asyncio.run(emit_request_outcome(handler, outcome))
+ assert handler.metrics.failed == []
+ assert len(handler.metrics.requested) == 1
diff --git a/tests/test_proxy_streaming_ratelimit_headers.py b/tests/test_proxy_streaming_ratelimit_headers.py
index 5983dea54..ae1f41862 100644
--- a/tests/test_proxy_streaming_ratelimit_headers.py
+++ b/tests/test_proxy_streaming_ratelimit_headers.py
@@ -293,6 +293,8 @@ class TestStreamingRatelimitHeaderForwarding:
proxy.http_client.send = AsyncMock(return_value=mock_response)
fake_logger = MagicMock()
monkeypatch.setattr(streaming_module, "logger", fake_logger)
+ prefix_tracker = MagicMock()
+ prefix_tracker.classify_cache_miss.return_value.is_miss = False
result = await proxy._stream_response(
url="https://api.anthropic.com/v1/messages",
@@ -312,6 +314,7 @@ class TestStreamingRatelimitHeaderForwarding:
transforms_applied=[],
tags={},
optimization_latency=0.0,
+ prefix_tracker=prefix_tracker,
)
assert result.status_code == 503
@@ -323,10 +326,56 @@ class TestStreamingRatelimitHeaderForwarding:
503,
"https://api.anthropic.com/v1/messages",
)
- proxy.metrics.record_request.assert_awaited_once()
- proxy.cost_tracker.record_tokens.assert_called_once()
+ # A 503 is an upstream failure, so it books via record_failed and stops
+ # before the savings/cost success path — otherwise a failed request
+ # inflates the save-rate (#1568).
+ proxy.metrics.record_failed.assert_awaited_once()
+ proxy.metrics.record_request.assert_not_awaited()
+ proxy.cost_tracker.record_tokens.assert_not_called()
+ prefix_tracker.classify_cache_miss.assert_not_called()
+ prefix_tracker.update_from_response.assert_not_called()
mock_response.aclose.assert_awaited_once()
+ @pytest.mark.asyncio
+ async def test_upstream_redirect_does_not_mutate_prefix_tracker(self):
+ """Upstream redirects should not change provider prefix state."""
+ proxy = self._create_mock_proxy()
+ mock_response = self._create_mock_upstream_response()
+ mock_response.status_code = 307
+
+ mock_request = MagicMock()
+ proxy.http_client.build_request = MagicMock(return_value=mock_request)
+ proxy.http_client.send = AsyncMock(return_value=mock_response)
+ prefix_tracker = MagicMock()
+ prefix_tracker.classify_cache_miss.return_value.is_miss = False
+
+ result = await proxy._stream_response(
+ url="https://api.anthropic.com/v1/messages",
+ headers={"x-api-key": "sk-test"},
+ body={
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 100,
+ "stream": True,
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ provider="anthropic",
+ model="claude-sonnet-4-20250514",
+ request_id="test-redirect",
+ original_tokens=10,
+ optimized_tokens=10,
+ tokens_saved=0,
+ transforms_applied=[],
+ tags={},
+ optimization_latency=0.0,
+ prefix_tracker=prefix_tracker,
+ )
+
+ async for _ in result.body_iterator:
+ pass
+
+ prefix_tracker.classify_cache_miss.assert_not_called()
+ prefix_tracker.update_from_response.assert_not_called()
+
@pytest.mark.asyncio
async def test_upstream_http_error_closes_response_when_body_read_fails(self, monkeypatch):
"""Reading a streaming error body should still close the upstream response."""