Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): count exhausted upstream 5xx as failed across all providers (#1571)
Rocker Zhange365ad715248
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 0573dbf32..d6820f680 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -3057,6 +3057,7 @@ class AnthropicHandlerMixin:request_id=request_id,provider=provider_name,model=model,+ status_code=response.status_code,original_tokens=original_tokens,optimized_tokens=optimized_tokens,output_tokens=output_tokens,@@ -3512,6 +3513,7 @@ class AnthropicHandlerMixin:request_id=request_id,provider="anthropic",model="batch",+ status_code=response.status_code,original_tokens=total_original_tokens,optimized_tokens=total_optimized_tokens,output_tokens=0,@@ -3636,6 +3638,7 @@ class AnthropicHandlerMixin:request_id=request_id,provider="anthropic",model="passthrough:batches",+ status_code=response.status_code,original_tokens=0,optimized_tokens=0,output_tokens=0,diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex c7163c0b9..666c7e906 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -423,6 +423,7 @@ class GeminiHandlerMixin:request_id=request_id,provider=provider_name,model=model,+ status_code=response.status_code,original_tokens=total_input_tokens,optimized_tokens=total_input_tokens,output_tokens=output_tokens,@@ -676,6 +677,7 @@ class GeminiHandlerMixin:request_id=request_id,provider=provider_name,model=model,+ status_code=response.status_code,original_tokens=original_tokens,optimized_tokens=total_input_tokens,output_tokens=output_tokens,@@ -1167,6 +1169,7 @@ class GeminiHandlerMixin:request_id=request_id,provider=provider_name,model=model,+ status_code=response.status_code,original_tokens=original_tokens,optimized_tokens=compressed_tokens,output_tokens=0,diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex e323aa6d1..7f33dc672 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3447,6 +3447,7 @@ class OpenAIHandlerMixin:request_id=request_id,provider=openai_chat_outcome_provider,model=model,+ status_code=response.status_code,original_tokens=original_tokens,optimized_tokens=total_input_tokens,output_tokens=output_tokens,@@ -4447,6 +4448,7 @@ class OpenAIHandlerMixin:request_id=request_id,provider="openai",model=model,+ status_code=response.status_code,original_tokens=effective_original_tokens,optimized_tokens=effective_optimized_tokens,output_tokens=output_tokens,@@ -7421,6 +7423,7 @@ class OpenAIHandlerMixin:request_id=request_id,provider=provider,model=_passthrough_model_from_path(path, endpoint_name),+ status_code=response.status_code,original_tokens=input_tokens,optimized_tokens=input_tokens,output_tokens=output_tokens,diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 7166a214f..750a2c0de 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -87,6 +87,12 @@ class RequestOutcome:# Prometheus ``cached`` counter and dashboard "response cache" row.from_response_cache: bool = False+ # Upstream HTTP status for this request (200 on success or response-cache+ # hit). When >= 500 (e.g. a 529 Overloaded returned after retry+ # exhaustion) the funnel records a failed request instead of feeding the+ # savings/cost stats, so an upstream failure can't inflate save-rate.+ status_code: int = 200+# ── Timing ────────────────────────────────────────────────────────# total_latency_ms: wall-clock end-to-end for this request# overhead_ms: time spent in compression dispatch only (subset of total)@@ -317,6 +323,10 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:(skipped when logger is None, i.e. ``--no-request-logging``)4. structured PERF log line — consumed by ``headroom perf``+ A failure outcome (``status_code >= 500``, e.g. a 529 surfaced after retry+ exhaustion) short-circuits before these four effects: it records a failed+ request and returns, so an upstream failure cannot feed the success stats.+Takes the handler as a free argument rather than ``self`` so thisfunction is callable from:* ``HeadroomProxy._record_request_outcome`` (production)@@ -333,6 +343,16 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:from headroom.proxy.models import RequestLogfrom headroom.proxy.project_context import get_current_project+ # Upstream failure (>= 500, e.g. a 529 Overloaded surfaced after retry+ # exhaustion) must not feed the savings/cost/log success stats; that would+ # let a failed request inflate the save-rate. Record it as failed and stop,+ # mirroring the pre-passthrough behaviour where an exhausted 5xx raised and+ # was counted via record_failed. 4xx stay on the normal funnel: they are+ # client errors the proxy still served.+ if outcome.status_code >= 500:+ await handler.metrics.record_failed(provider=outcome.provider)+ return+# Output-shaping savings ledger (counterfactual estimator). The shaper# tags each request's (arm, stratum) onto ``transforms_applied``; feed the# observed output tokens to the recorder so it can produce an honestdiff --git a/tests/test_5xx_accounting_all_providers.py b/tests/test_5xx_accounting_all_providers.pynew file mode 100644index 000000000..0dd489f53--- /dev/null+++ b/tests/test_5xx_accounting_all_providers.py@@ -0,0 +1,184 @@+"""5xx accounting for the OpenAI and Gemini handler paths.++The >=500 routing in emit_request_outcome is covered provider-agnostically by+tests/test_outcome_records_5xx_as_failed.py. This file pins the per-provider+contract: every retry-fed handler site now threads the real upstream status+onto RequestOutcome (mirroring the Anthropic sites), so an exhausted 5xx is+recorded as failed and skips the savings/cost funnel rather than inflating the+save-rate. Each case below corresponds to one wired call site.+"""++import asyncio++import pytest++from headroom.proxy.outcome import RequestOutcome, emit_request_outcome+++class _Metrics:+ def __init__(self):+ self.failed = []+ self.requested = []++ async def record_failed(self, provider):+ self.failed.append(provider)++ async def record_request(self, **kwargs):+ self.requested.append(kwargs)+++class _Handler:+ # Minimal stub: if emit_request_outcome escapes the >=500 guard it reaches+ # the success funnel and AttributeErrors on cost_tracker/logger.+ def __init__(self):+ self.metrics = _Metrics()+++# (label, provider, status) — one entry per retry-fed site wired in this change.+WIRED_SITES = [+ ("openai.chat", "openai", 529),+ ("openai.chat", "openai", 503),+ ("openai.responses", "openai", 503),+ ("openai.passthrough", "anthropic", 503),+ ("gemini.generateContent", "gemini", 529),+ ("gemini.generateContent", "gemini", 503),+ ("gemini.allNonText", "gemini", 503),+ ("gemini.countTokens", "gemini", 503),+]++[email protected]("label,provider,status", WIRED_SITES)+def test_exhausted_5xx_recorded_as_failed(label, provider, status):+ handler = _Handler()+ outcome = RequestOutcome(+ request_id=f"req-{label}",+ provider=provider,+ model="test-model",+ status_code=status,+ original_tokens=100,+ optimized_tokens=100,+ output_tokens=0,+ tokens_saved=0,+ attempted_input_tokens=100,+ )+ asyncio.run(emit_request_outcome(handler, outcome))+ assert handler.metrics.failed == [provider]+ assert handler.metrics.requested == []+++# --- handler-level wiring test -------------------------------------------+#+# The parametrized cases above hand-build RequestOutcome, so they would not+# catch a regression where a handler stops passing status_code=response.+# status_code. This test invokes a real handler entry method end to end with a+# transport that exhausts retries on 503, then asserts the outcome the handler+# emitted carries that 503 — i.e. deleting the wiring would fail here.+#+# gemini countTokens is the cheapest retry-fed path: it never streams, has no+# cache, and skips the compression pipeline under optimize=False.+#+# Status note: _retry_request (server.py) returns a 429/529 *verbatim* on+# exhaustion (RETRYABLE_OVERLOAD_STATUSES) but re-raises a generic 503 as+# HTTPStatusError. So the response-fed RequestOutcome is only reachable for the+# overload statuses — 529 is the live scenario this change targets ("529+# Overloaded surfaced after retry exhaustion").+++class _Always529Transport:+ """httpx transport returning 529 every call (overload, exhausts retries)."""++ def __init__(self):+ self.calls = 0++ async def handle_async_request(self, request):+ self.calls += 1+ async for _ in request.stream: # drain body+ pass+ return _httpx().Response(529, json={"error": {"message": "overloaded"}})++ async def aclose(self): # let AsyncClient.aclose() tear down cleanly+ pass+++def _httpx():+ import httpx++ return httpx+++def _count_tokens_request(body_bytes: bytes):+ """Real Starlette Request over an ASGI scope (exercises body parsing)."""+ from starlette.requests import Request++ scope = {+ "type": "http",+ "method": "POST",+ "path": "/v1beta/models/gemini-2.0-flash:countTokens",+ "raw_path": b"/v1beta/models/gemini-2.0-flash:countTokens",+ "query_string": b"",+ "headers": [(b"content-type", b"application/json")],+ }+ sent = {"done": False}++ async def receive():+ if sent["done"]:+ return {"type": "http.disconnect"}+ sent["done"] = True+ return {"type": "http.request", "body": body_bytes, "more_body": False}++ return Request(scope, receive)+++def test_gemini_count_tokens_handler_threads_real_529_onto_outcome():+ import json++ from headroom.proxy.server import ProxyConfig, create_app++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ retry_enabled=True,+ retry_max_attempts=2,+ retry_base_delay_ms=1,+ retry_max_delay_ms=5,+ )+ proxy = create_app(config).state.proxy+ original_client = proxy.http_client+ transport = _Always529Transport()+ injected_client = _httpx().AsyncClient(transport=transport)+ proxy.http_client = injected_client++ captured = []++ async def _capture(outcome):+ captured.append(outcome)++ proxy._record_request_outcome = _capture # capture, skip the funnel++ body = json.dumps({"contents": [{"role": "user", "parts": [{"text": "hello world"}]}]}).encode()+ request = _count_tokens_request(body)++ async def _run():+ try:+ await proxy.handle_gemini_count_tokens(request, model="gemini-2.0-flash")+ finally:+ await injected_client.aclose()+ if original_client is not None:+ await original_client.aclose()++ asyncio.run(_run())++ # retry_max_attempts=2, so an exhausted 529 means exactly 2 upstream calls;+ # asserting the count guards the retry-exhaustion path the test covers.+ assert transport.calls == 2+ assert captured, "handler did not emit a RequestOutcome"+ # The crux: the real upstream 529 is threaded onto the outcome. If the+ # handler dropped status_code=response.status_code this would be 200.+ assert captured[-1].status_code == 529diff --git a/tests/test_outcome_records_5xx_as_failed.py b/tests/test_outcome_records_5xx_as_failed.pynew file mode 100644index 000000000..5e6b95657--- /dev/null+++ b/tests/test_outcome_records_5xx_as_failed.py@@ -0,0 +1,59 @@+"""An exhausted-5xx outcome (e.g. a 529 Overloaded surfaced after retry+exhaustion) must be counted as a failed request, not fed into the+savings/cost success funnel where it would inflate the save-rate.++Companion to the retry-exhaustion change that returns the real upstream 5xx+instead of collapsing it to a 502.+"""++import asyncio++from headroom.proxy.outcome import RequestOutcome, emit_request_outcome+++class _Metrics:+ def __init__(self):+ self.failed = []+ self.requested = []++ async def record_failed(self, provider):+ self.failed.append(provider)++ async def record_request(self, **kwargs):+ self.requested.append(kwargs)+++class _Handler:+ # Deliberately exposes ONLY .metrics. If emit_request_outcome runs past the+ # >=500 guard it will AttributeError on cost_tracker/logger, failing the+ # test — which is exactly the contract we want to lock in for 5xx.+ def __init__(self):+ self.metrics = _Metrics()+++def _outcome(status_code):+ return RequestOutcome(+ request_id="req-1",+ provider="anthropic",+ model="claude-opus-4-8",+ original_tokens=0,+ optimized_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ attempted_input_tokens=0,+ status_code=status_code,+ )+++def test_529_recorded_as_failed_and_skips_success_funnel():+ handler = _Handler()+ asyncio.run(emit_request_outcome(handler, _outcome(529)))+ assert handler.metrics.failed == ["anthropic"] # counted as failed+ assert handler.metrics.requested == [] # NOT counted as a served request+++def test_503_recorded_as_failed():+ handler = _Handler()+ asyncio.run(emit_request_outcome(handler, _outcome(503)))+ assert handler.metrics.failed == ["anthropic"]+ assert handler.metrics.requested == []