Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): count exhausted upstream 5xx as failed across all providers (#1571)

Rocker Zhange365ad715248
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 0573dbf32..d6820f680 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -3057,6 +3057,7 @@ class AnthropicHandlerMixin:
request_id=request_id,
provider=provider_name,
model=model,
+ status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=optimized_tokens,
output_tokens=output_tokens,
@@ -3512,6 +3513,7 @@ class AnthropicHandlerMixin:
request_id=request_id,
provider="anthropic",
model="batch",
+ status_code=response.status_code,
original_tokens=total_original_tokens,
optimized_tokens=total_optimized_tokens,
output_tokens=0,
@@ -3636,6 +3638,7 @@ class AnthropicHandlerMixin:
request_id=request_id,
provider="anthropic",
model="passthrough:batches",
+ status_code=response.status_code,
original_tokens=0,
optimized_tokens=0,
output_tokens=0,
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
index c7163c0b9..666c7e906 100644
--- a/headroom/proxy/handlers/gemini.py
+++ b/headroom/proxy/handlers/gemini.py
@@ -423,6 +423,7 @@ class GeminiHandlerMixin:
request_id=request_id,
provider=provider_name,
model=model,
+ status_code=response.status_code,
original_tokens=total_input_tokens,
optimized_tokens=total_input_tokens,
output_tokens=output_tokens,
@@ -676,6 +677,7 @@ class GeminiHandlerMixin:
request_id=request_id,
provider=provider_name,
model=model,
+ status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=total_input_tokens,
output_tokens=output_tokens,
@@ -1167,6 +1169,7 @@ class GeminiHandlerMixin:
request_id=request_id,
provider=provider_name,
model=model,
+ status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=compressed_tokens,
output_tokens=0,
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index e323aa6d1..7f33dc672 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3447,6 +3447,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
provider=openai_chat_outcome_provider,
model=model,
+ status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=total_input_tokens,
output_tokens=output_tokens,
@@ -4447,6 +4448,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
provider="openai",
model=model,
+ status_code=response.status_code,
original_tokens=effective_original_tokens,
optimized_tokens=effective_optimized_tokens,
output_tokens=output_tokens,
@@ -7421,6 +7423,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
provider=provider,
model=_passthrough_model_from_path(path, endpoint_name),
+ status_code=response.status_code,
original_tokens=input_tokens,
optimized_tokens=input_tokens,
output_tokens=output_tokens,
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 7166a214f..750a2c0de 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -87,6 +87,12 @@ class RequestOutcome:
# Prometheus ``cached`` counter and dashboard "response cache" row.
from_response_cache: bool = False
+ # Upstream HTTP status for this request (200 on success or response-cache
+ # hit). When >= 500 (e.g. a 529 Overloaded returned after retry
+ # exhaustion) the funnel records a failed request instead of feeding the
+ # savings/cost stats, so an upstream failure can't inflate save-rate.
+ status_code: int = 200
+
# ── Timing ────────────────────────────────────────────────────────
# total_latency_ms: wall-clock end-to-end for this request
# overhead_ms: time spent in compression dispatch only (subset of total)
@@ -317,6 +323,10 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
(skipped when logger is None, i.e. ``--no-request-logging``)
4. structured PERF log line — consumed by ``headroom perf``
+ A failure outcome (``status_code >= 500``, e.g. a 529 surfaced after retry
+ exhaustion) short-circuits before these four effects: it records a failed
+ request and returns, so an upstream failure cannot feed the success stats.
+
Takes the handler as a free argument rather than ``self`` so this
function is callable from:
* ``HeadroomProxy._record_request_outcome`` (production)
@@ -333,6 +343,16 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
from headroom.proxy.models import RequestLog
from headroom.proxy.project_context import get_current_project
+ # Upstream failure (>= 500, e.g. a 529 Overloaded surfaced after retry
+ # exhaustion) must not feed the savings/cost/log success stats; that would
+ # let a failed request inflate the save-rate. Record it as failed and stop,
+ # mirroring the pre-passthrough behaviour where an exhausted 5xx raised and
+ # was counted via record_failed. 4xx stay on the normal funnel: they are
+ # client errors the proxy still served.
+ if outcome.status_code >= 500:
+ await handler.metrics.record_failed(provider=outcome.provider)
+ return
+
# Output-shaping savings ledger (counterfactual estimator). The shaper
# tags each request's (arm, stratum) onto ``transforms_applied``; feed the
# observed output tokens to the recorder so it can produce an honest
diff --git a/tests/test_5xx_accounting_all_providers.py b/tests/test_5xx_accounting_all_providers.py
new file mode 100644
index 000000000..0dd489f53
--- /dev/null
+++ b/tests/test_5xx_accounting_all_providers.py
@@ -0,0 +1,184 @@
+"""5xx accounting for the OpenAI and Gemini handler paths.
+
+The >=500 routing in emit_request_outcome is covered provider-agnostically by
+tests/test_outcome_records_5xx_as_failed.py. This file pins the per-provider
+contract: every retry-fed handler site now threads the real upstream status
+onto RequestOutcome (mirroring the Anthropic sites), so an exhausted 5xx is
+recorded as failed and skips the savings/cost funnel rather than inflating the
+save-rate. Each case below corresponds to one wired call site.
+"""
+
+import asyncio
+
+import pytest
+
+from headroom.proxy.outcome import RequestOutcome, emit_request_outcome
+
+
+class _Metrics:
+ def __init__(self):
+ self.failed = []
+ self.requested = []
+
+ async def record_failed(self, provider):
+ self.failed.append(provider)
+
+ async def record_request(self, **kwargs):
+ self.requested.append(kwargs)
+
+
+class _Handler:
+ # Minimal stub: if emit_request_outcome escapes the >=500 guard it reaches
+ # the success funnel and AttributeErrors on cost_tracker/logger.
+ def __init__(self):
+ self.metrics = _Metrics()
+
+
+# (label, provider, status) — one entry per retry-fed site wired in this change.
+WIRED_SITES = [
+ ("openai.chat", "openai", 529),
+ ("openai.chat", "openai", 503),
+ ("openai.responses", "openai", 503),
+ ("openai.passthrough", "anthropic", 503),
+ ("gemini.generateContent", "gemini", 529),
+ ("gemini.generateContent", "gemini", 503),
+ ("gemini.allNonText", "gemini", 503),
+ ("gemini.countTokens", "gemini", 503),
+]
+
+
[email protected]("label,provider,status", WIRED_SITES)
+def test_exhausted_5xx_recorded_as_failed(label, provider, status):
+ handler = _Handler()
+ outcome = RequestOutcome(
+ request_id=f"req-{label}",
+ provider=provider,
+ model="test-model",
+ status_code=status,
+ original_tokens=100,
+ optimized_tokens=100,
+ output_tokens=0,
+ tokens_saved=0,
+ attempted_input_tokens=100,
+ )
+ asyncio.run(emit_request_outcome(handler, outcome))
+ assert handler.metrics.failed == [provider]
+ assert handler.metrics.requested == []
+
+
+# --- handler-level wiring test -------------------------------------------
+#
+# The parametrized cases above hand-build RequestOutcome, so they would not
+# catch a regression where a handler stops passing status_code=response.
+# status_code. This test invokes a real handler entry method end to end with a
+# transport that exhausts retries on 503, then asserts the outcome the handler
+# emitted carries that 503 — i.e. deleting the wiring would fail here.
+#
+# gemini countTokens is the cheapest retry-fed path: it never streams, has no
+# cache, and skips the compression pipeline under optimize=False.
+#
+# Status note: _retry_request (server.py) returns a 429/529 *verbatim* on
+# exhaustion (RETRYABLE_OVERLOAD_STATUSES) but re-raises a generic 503 as
+# HTTPStatusError. So the response-fed RequestOutcome is only reachable for the
+# overload statuses — 529 is the live scenario this change targets ("529
+# Overloaded surfaced after retry exhaustion").
+
+
+class _Always529Transport:
+ """httpx transport returning 529 every call (overload, exhausts retries)."""
+
+ def __init__(self):
+ self.calls = 0
+
+ async def handle_async_request(self, request):
+ self.calls += 1
+ async for _ in request.stream: # drain body
+ pass
+ return _httpx().Response(529, json={"error": {"message": "overloaded"}})
+
+ async def aclose(self): # let AsyncClient.aclose() tear down cleanly
+ pass
+
+
+def _httpx():
+ import httpx
+
+ return httpx
+
+
+def _count_tokens_request(body_bytes: bytes):
+ """Real Starlette Request over an ASGI scope (exercises body parsing)."""
+ from starlette.requests import Request
+
+ scope = {
+ "type": "http",
+ "method": "POST",
+ "path": "/v1beta/models/gemini-2.0-flash:countTokens",
+ "raw_path": b"/v1beta/models/gemini-2.0-flash:countTokens",
+ "query_string": b"",
+ "headers": [(b"content-type", b"application/json")],
+ }
+ sent = {"done": False}
+
+ async def receive():
+ if sent["done"]:
+ return {"type": "http.disconnect"}
+ sent["done"] = True
+ return {"type": "http.request", "body": body_bytes, "more_body": False}
+
+ return Request(scope, receive)
+
+
+def test_gemini_count_tokens_handler_threads_real_529_onto_outcome():
+ import json
+
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ retry_enabled=True,
+ retry_max_attempts=2,
+ retry_base_delay_ms=1,
+ retry_max_delay_ms=5,
+ )
+ proxy = create_app(config).state.proxy
+ original_client = proxy.http_client
+ transport = _Always529Transport()
+ injected_client = _httpx().AsyncClient(transport=transport)
+ proxy.http_client = injected_client
+
+ captured = []
+
+ async def _capture(outcome):
+ captured.append(outcome)
+
+ proxy._record_request_outcome = _capture # capture, skip the funnel
+
+ body = json.dumps({"contents": [{"role": "user", "parts": [{"text": "hello world"}]}]}).encode()
+ request = _count_tokens_request(body)
+
+ async def _run():
+ try:
+ await proxy.handle_gemini_count_tokens(request, model="gemini-2.0-flash")
+ finally:
+ await injected_client.aclose()
+ if original_client is not None:
+ await original_client.aclose()
+
+ asyncio.run(_run())
+
+ # retry_max_attempts=2, so an exhausted 529 means exactly 2 upstream calls;
+ # asserting the count guards the retry-exhaustion path the test covers.
+ assert transport.calls == 2
+ assert captured, "handler did not emit a RequestOutcome"
+ # The crux: the real upstream 529 is threaded onto the outcome. If the
+ # handler dropped status_code=response.status_code this would be 200.
+ assert captured[-1].status_code == 529
diff --git a/tests/test_outcome_records_5xx_as_failed.py b/tests/test_outcome_records_5xx_as_failed.py
new file mode 100644
index 000000000..5e6b95657
--- /dev/null
+++ b/tests/test_outcome_records_5xx_as_failed.py
@@ -0,0 +1,59 @@
+"""An exhausted-5xx outcome (e.g. a 529 Overloaded surfaced after retry
+exhaustion) must be counted as a failed request, not fed into the
+savings/cost success funnel where it would inflate the save-rate.
+
+Companion to the retry-exhaustion change that returns the real upstream 5xx
+instead of collapsing it to a 502.
+"""
+
+import asyncio
+
+from headroom.proxy.outcome import RequestOutcome, emit_request_outcome
+
+
+class _Metrics:
+ def __init__(self):
+ self.failed = []
+ self.requested = []
+
+ async def record_failed(self, provider):
+ self.failed.append(provider)
+
+ async def record_request(self, **kwargs):
+ self.requested.append(kwargs)
+
+
+class _Handler:
+ # Deliberately exposes ONLY .metrics. If emit_request_outcome runs past the
+ # >=500 guard it will AttributeError on cost_tracker/logger, failing the
+ # test — which is exactly the contract we want to lock in for 5xx.
+ def __init__(self):
+ self.metrics = _Metrics()
+
+
+def _outcome(status_code):
+ return RequestOutcome(
+ request_id="req-1",
+ provider="anthropic",
+ model="claude-opus-4-8",
+ original_tokens=0,
+ optimized_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ attempted_input_tokens=0,
+ status_code=status_code,
+ )
+
+
+def test_529_recorded_as_failed_and_skips_success_funnel():
+ handler = _Handler()
+ asyncio.run(emit_request_outcome(handler, _outcome(529)))
+ assert handler.metrics.failed == ["anthropic"] # counted as failed
+ assert handler.metrics.requested == [] # NOT counted as a served request
+
+
+def test_503_recorded_as_failed():
+ handler = _Handler()
+ asyncio.run(emit_request_outcome(handler, _outcome(503)))
+ assert handler.metrics.failed == ["anthropic"]
+ assert handler.metrics.requested == []