Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): record streaming upstream 5xx as failed instead of success (#2682)
Omar Gerardo4949cd5594f4
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 7e8d49f7c..66e89fae3 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -931,6 +931,7 @@ class StreamingMixin:parsed_response: dict[str, Any] | None = None,client: str | None = None,waste_signals: dict[str, int] | None = None,+ status_code: int = 200,# Set only by paths whose ``tokens_saved`` is the CONVERSATION's# running total rather than this turn's -- OpenAI ``/v1/responses``,# which re-sends and recompresses the whole transcript every turn.@@ -1025,7 +1026,7 @@ class StreamingMixin:# Prefix-tracker mutation is provider-specific state that lives# outside the metric funnel. Run it before the funnel so the next# request inherits correct prefix state regardless of metric path.- if prefix_tracker is not None:+ if 200 <= status_code < 300 and prefix_tracker is not None:import copy as _copyforwarded_messages = body.get("messages", [])@@ -1108,6 +1109,7 @@ class StreamingMixin:overhead_ms=optimization_latency,tags=outcome_tags,client=client,+ status_code=status_code,log_full_messages=getattr(self.config, "log_full_messages", False),cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,@@ -1532,6 +1534,7 @@ class StreamingMixin:original_messages=original_messages,client=client,waste_signals=waste_signals,+ status_code=upstream_response.status_code,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,)@@ -1813,6 +1816,7 @@ class StreamingMixin:parsed_response=parsed_response,client=client,waste_signals=waste_signals,+ status_code=upstream_response.status_code,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,)diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 3f0897f56..545cf9f73 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -319,6 +319,7 @@ class RequestOutcome:overhead_ms: float,tags: dict[str, str] | None,client: str | None,+ status_code: int = 200,log_full_messages: bool = False,cache_read_tokens: int = 0,cache_write_tokens: int = 0,@@ -411,6 +412,7 @@ class RequestOutcome:conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,attempted_input_tokens=optimized_tokens + tokens_saved,+ status_code=status_code,cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,cache_write_5m_tokens=cache_write_5m_tokens,diff --git a/tests/test_outcome_records_5xx_as_failed.py b/tests/test_outcome_records_5xx_as_failed.pyindex 5e6b95657..0c2203b31 100644--- a/tests/test_outcome_records_5xx_as_failed.py+++ b/tests/test_outcome_records_5xx_as_failed.py@@ -57,3 +57,52 @@ def test_503_recorded_as_failed():asyncio.run(emit_request_outcome(handler, _outcome(503)))assert handler.metrics.failed == ["anthropic"]assert handler.metrics.requested == []+++def test_stream_outcome_529_recorded_as_failed_and_skips_success_funnel():+ handler = _Handler()+ outcome = RequestOutcome.from_stream(+ body={"messages": []},+ provider="anthropic",+ model="claude-opus-4-8",+ request_id="req-stream-529",+ original_tokens=100,+ optimized_tokens=80,+ output_tokens=0,+ tokens_saved=20,+ transforms_applied=[],+ total_latency_ms=100.0,+ overhead_ms=10.0,+ tags=None,+ client=None,+ status_code=529,+ )++ assert outcome.status_code == 529+ asyncio.run(emit_request_outcome(handler, outcome))+ assert handler.metrics.failed == ["anthropic"]+ assert handler.metrics.requested == []+++def test_stream_outcome_without_status_code_defaults_to_success():+ handler = _Handler()+ outcome = RequestOutcome.from_stream(+ body={"messages": []},+ provider="anthropic",+ model="claude-opus-4-8",+ request_id="req-stream-200",+ original_tokens=100,+ optimized_tokens=80,+ output_tokens=10,+ tokens_saved=20,+ transforms_applied=[],+ total_latency_ms=100.0,+ overhead_ms=10.0,+ tags=None,+ client=None,+ )++ assert outcome.status_code == 200+ asyncio.run(emit_request_outcome(handler, outcome))+ assert handler.metrics.failed == []+ assert len(handler.metrics.requested) == 1diff --git a/tests/test_proxy_streaming_ratelimit_headers.py b/tests/test_proxy_streaming_ratelimit_headers.pyindex 5983dea54..ae1f41862 100644--- a/tests/test_proxy_streaming_ratelimit_headers.py+++ b/tests/test_proxy_streaming_ratelimit_headers.py@@ -293,6 +293,8 @@ class TestStreamingRatelimitHeaderForwarding:proxy.http_client.send = AsyncMock(return_value=mock_response)fake_logger = MagicMock()monkeypatch.setattr(streaming_module, "logger", fake_logger)+ prefix_tracker = MagicMock()+ prefix_tracker.classify_cache_miss.return_value.is_miss = Falseresult = await proxy._stream_response(url="https://api.anthropic.com/v1/messages",@@ -312,6 +314,7 @@ class TestStreamingRatelimitHeaderForwarding:transforms_applied=[],tags={},optimization_latency=0.0,+ prefix_tracker=prefix_tracker,)assert result.status_code == 503@@ -323,10 +326,56 @@ class TestStreamingRatelimitHeaderForwarding:503,"https://api.anthropic.com/v1/messages",)- proxy.metrics.record_request.assert_awaited_once()- proxy.cost_tracker.record_tokens.assert_called_once()+ # A 503 is an upstream failure, so it books via record_failed and stops+ # before the savings/cost success path — otherwise a failed request+ # inflates the save-rate (#1568).+ proxy.metrics.record_failed.assert_awaited_once()+ proxy.metrics.record_request.assert_not_awaited()+ proxy.cost_tracker.record_tokens.assert_not_called()+ prefix_tracker.classify_cache_miss.assert_not_called()+ prefix_tracker.update_from_response.assert_not_called()mock_response.aclose.assert_awaited_once()+ @pytest.mark.asyncio+ async def test_upstream_redirect_does_not_mutate_prefix_tracker(self):+ """Upstream redirects should not change provider prefix state."""+ proxy = self._create_mock_proxy()+ mock_response = self._create_mock_upstream_response()+ mock_response.status_code = 307++ mock_request = MagicMock()+ proxy.http_client.build_request = MagicMock(return_value=mock_request)+ proxy.http_client.send = AsyncMock(return_value=mock_response)+ prefix_tracker = MagicMock()+ prefix_tracker.classify_cache_miss.return_value.is_miss = False++ result = await proxy._stream_response(+ url="https://api.anthropic.com/v1/messages",+ headers={"x-api-key": "sk-test"},+ body={+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 100,+ "stream": True,+ "messages": [{"role": "user", "content": "hi"}],+ },+ provider="anthropic",+ model="claude-sonnet-4-20250514",+ request_id="test-redirect",+ original_tokens=10,+ optimized_tokens=10,+ tokens_saved=0,+ transforms_applied=[],+ tags={},+ optimization_latency=0.0,+ prefix_tracker=prefix_tracker,+ )++ async for _ in result.body_iterator:+ pass++ prefix_tracker.classify_cache_miss.assert_not_called()+ prefix_tracker.update_from_response.assert_not_called()+@pytest.mark.asyncioasync def test_upstream_http_error_closes_response_when_body_read_fails(self, monkeypatch):"""Reading a streaming error body should still close the upstream response."""