Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/openai): propagate provider usage on the Responses WS->HTTP fallback (#2988)
Abhay Singh536c949a692f
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 1ced3014b..a73ac0ff7 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -8602,9 +8602,23 @@ class OpenAIHandlerMixin:f"[{request_id}] WS upstream failed ({_ws_detail}), "f"falling back to HTTP POST streaming")- await self._ws_http_fallback(+ (+ fb_input_tokens,+ fb_output_tokens,+ fb_cache_read_tokens,+ fb_cache_write_tokens,+ fb_uncached_tokens,+ ) = await self._ws_http_fallback(websocket, body, first_msg_raw, upstream_headers, request_id)+ # Fold the fallback's provider usage into the session totals so+ # the WS session-end outcome records the authoritative wire-token+ # count instead of 0 (#2957).+ ws_input_tokens_total += fb_input_tokens+ ws_output_tokens_total += fb_output_tokens+ ws_cache_read_tokens_total += fb_cache_read_tokens+ ws_cache_write_tokens_total += fb_cache_write_tokens+ ws_uncached_input_tokens_total += fb_uncached_tokens# ── WS session-end metric + RequestLog ──────────────────#@@ -8850,14 +8864,31 @@ class OpenAIHandlerMixin:first_msg_raw: str,upstream_headers: dict[str, str],request_id: str,- ) -> None:+ ) -> tuple[int, int, int, int, int]:"""Fall back to HTTP POST streaming when upstream WS fails.Converts the WS ``response.create`` message to an HTTP POST to``/v1/responses?stream=true``, reads SSE events, and relays each``data:`` line as a WS text message to the client. This makesCodex work immediately instead of exhausting its WS retry budget.++ Returns ``(input, output, cache_read, cache_write, uncached)`` provider+ usage parsed from the ``response.completed`` SSE event. The caller folds+ it into the session totals so the WS session-end outcome uses the+ authoritative wire-token count; otherwise a fallback recorded+ ``input_tokens=0`` and savings percentages blew past 100 (#2957)."""+ fallback_usage = [0, 0, 0, 0, 0]++ def _accumulate_usage(data_str: str) -> None:+ try:+ event = json.loads(data_str)+ except (json.JSONDecodeError, TypeError):+ return+ if isinstance(event, dict) and event.get("type") == "response.completed":+ for i, value in enumerate(_extract_responses_usage(event)):+ fallback_usage[i] += value+# Route to correct endpoint based on auth modeis_chatgpt_fallback = has_chatgpt_account_header(upstream_headers)if is_chatgpt_fallback:@@ -8963,7 +8994,7 @@ class OpenAIHandlerMixin:},}await websocket.send_text(json.dumps(error_event))- return+ return tuple(fallback_usage) # type: ignore[return-value]# Refresh Codex /stats from the fallback response# headers. We can't forward them onto the client 101@@ -8989,10 +9020,11 @@ class OpenAIHandlerMixin:data = line[6:]if data == "[DONE]":continue+ _accumulate_usage(data)try:await websocket.send_text(data)except Exception:- return+ return tuple(fallback_usage) # type: ignore[return-value]elif line.startswith("event: "):# SSE event type — skip, the data line contains the typecontinue@@ -9001,9 +9033,10 @@ class OpenAIHandlerMixin:for line in buffer.strip().splitlines():line = line.strip()if line.startswith("data: ") and line[6:] != "[DONE]":+ _accumulate_usage(line[6:])with contextlib.suppress(Exception):await websocket.send_text(line[6:])- return+ return tuple(fallback_usage) # type: ignore[return-value]except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:if http_attempt >= retry_attempts - 1:raise@@ -9034,6 +9067,7 @@ class OpenAIHandlerMixin:finally:with contextlib.suppress(Exception):await websocket.close()+ return tuple(fallback_usage) # type: ignore[return-value]def _derived_compress_pipeline(self, key: str, **overrides: Any) -> Any:"""Cached ``/v1/compress`` pipeline derived from the live OpenAI router.diff --git a/tests/test_ws_http_fallback.py b/tests/test_ws_http_fallback.pyindex 459246fb3..d0205c877 100644--- a/tests/test_ws_http_fallback.py+++ b/tests/test_ws_http_fallback.py@@ -310,6 +310,53 @@ class TestWsHttpFallback:assert "api.openai.com" in captured_url["url"]+ def test_fallback_returns_provider_usage_from_completed_event(self):+ """The fallback must surface the provider's input usage (#2957).++ Otherwise the WS session-end outcome records input_tokens=0 for a large+ request and savings percentages blow past 100.+ """+ handler = _make_handler()+ ws = FakeWebSocket()+ completed = {+ "type": "response.completed",+ "response": {+ "usage": {+ "input_tokens": 31055,+ "output_tokens": 246,+ "input_tokens_details": {"cached_tokens": 20000},+ }+ },+ }+ sse_lines = [+ 'data: {"type":"response.created","response":{"id":"r1"}}\n\n',+ f"data: {json.dumps(completed)}\n\n",+ "data: [DONE]\n\n",+ ]+ handler.http_client = FakeHttpClient(FakeStreamResponse(200, sse_lines))++ body = {"model": "gpt-5.4", "input": "big context"}+ usage = asyncio.run(handler._ws_http_fallback(ws, body, json.dumps(body), {}, "req_usage"))++ input_tokens, output_tokens, cache_read, _cache_write, uncached = usage+ assert input_tokens == 31055+ assert output_tokens == 246+ assert cache_read == 20000+ assert uncached == 31055 - 20000++ def test_fallback_returns_zero_usage_without_completed_event(self):+ handler = _make_handler()+ ws = FakeWebSocket()+ handler.http_client = FakeHttpClient(+ FakeStreamResponse(200, ['data: {"type":"response.created"}\n\n', "data: [DONE]\n\n"])+ )+ usage = asyncio.run(+ handler._ws_http_fallback(+ ws, {"model": "gpt-5.4", "input": "hi"}, json.dumps({"input": "hi"}), {}, "req_none"+ )+ )+ assert usage == (0, 0, 0, 0, 0)+def test_fallback_refreshes_codex_rate_limit_state(self, monkeypatch):"""A successful fallback refreshes Codex /stats from response headers.