Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): bound Codex Responses compression work
Gili Tzabari160989c43ecb
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 08fc3baef..fe12b7d9d 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -18,7 +18,11 @@ import uuidfrom datetime import datetimefrom typing import TYPE_CHECKING, Any-from headroom.proxy.helpers import _headroom_bypass_enabled, jitter_delay_ms+from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS,+ _headroom_bypass_enabled,+ jitter_delay_ms,+)from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_logfrom headroom.proxy.ws_session_registry import (TerminationCause,@@ -867,6 +871,22 @@ class OpenAIHandlerMixin:attempted_input_tokens,)+ async def _compress_openai_responses_payload_in_executor(+ self,+ payload: dict[str, Any],+ *,+ model: str,+ request_id: str,+ ) -> tuple[dict[str, Any], bool, int, list[str], str | None, int, int, int]:+ return await self._run_compression_in_executor(+ lambda: self._compress_openai_responses_payload(+ payload,+ model=model,+ request_id=request_id,+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )+async def handle_openai_chat(self,request: Request,@@ -2346,7 +2366,7 @@ class OpenAIHandlerMixin:_bytes_before,_bytes_after,_attempted_tokens,- ) = self._compress_openai_responses_payload(+ ) = await self._compress_openai_responses_payload_in_executor(body,model=model,request_id=request_id,@@ -3253,7 +3273,7 @@ class OpenAIHandlerMixin:_bytes_before,_bytes_after,_ws_attempted_tokens,- ) = self._compress_openai_responses_payload(+ ) = await self._compress_openai_responses_payload_in_executor(_inner,model=_model,request_id=request_id,@@ -3477,7 +3497,7 @@ class OpenAIHandlerMixin:bytes_before,bytes_after,frame_attempted_tokens,- ) = self._compress_openai_responses_payload(+ ) = await self._compress_openai_responses_payload_in_executor(inner_payload,model=model_for_frame,request_id=request_id,@@ -4785,12 +4805,33 @@ class OpenAIHandlerMixin:body = await request.body()headers = await apply_copilot_api_auth(headers, url=url)- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,- )+ try:+ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )+ except (httpx.ConnectError, httpx.TimeoutException) as e:+ logger.warning(+ "Passthrough request failed before upstream response: %s %s -> %s: %s",+ request.method,+ path,+ url,+ e,+ )+ return Response(+ content=json.dumps(+ {+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {e}",+ }+ }+ ),+ status_code=502,+ media_type="application/json",+ )# Remove compression headers since httpx already decompressed the responseresponse_headers = dict(response.headers)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 2ec8f2723..8877a1775 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1171,7 +1171,7 @@ class HeadroomProxy(return response- except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:+ except (httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError) as e:last_error = eif not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:diff --git a/tests/test_openai_codex_ws_lifecycle.py b/tests/test_openai_codex_ws_lifecycle.pyindex d9bd3f316..f62e68078 100644--- a/tests/test_openai_codex_ws_lifecycle.py+++ b/tests/test_openai_codex_ws_lifecycle.py@@ -15,6 +15,7 @@ from unittest.mock import MagicMock, patchimport pytest+from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDSfrom headroom.proxy.handlers.openai import OpenAIHandlerMixinfrom headroom.proxy.ws_session_registry import WebSocketSessionRegistry@@ -78,10 +79,17 @@ class _DummyOpenAIHandler(OpenAIHandlerMixin):self.cost_tracker = Noneself.memory_handler = Noneself.ws_sessions = ws_sessions or WebSocketSessionRegistry()+ self.compression_executor_calls = 0+ self.compression_executor_timeouts: list[float] = []async def _next_request_id(self) -> str:return "req-lifecycle-test"+ async def _run_compression_in_executor(self, fn, *, timeout: float):+ self.compression_executor_calls += 1+ self.compression_executor_timeouts.append(timeout)+ return fn()+class _FakeWebSocketDisconnect(Exception):"""Mirrors the ``WebSocketDisconnect`` type-name check in the handler.@@ -224,6 +232,39 @@ def _first_frame() -> str:# ---------------------------------------------------------------------------+async def test_ws_first_frame_compression_uses_bounded_executor():+ """Codex WS compression must not run synchronously on the event loop."""+ upstream_events = [+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),+ json.dumps({"type": "response.completed", "response": {"id": "r_1"}}),+ ]+ upstream = _FakeUpstream(upstream_events)+ fake_ws_mod = _make_fake_websockets_module(upstream)++ client_ws = _FakeWebSocket(frames=[_first_frame()])+ handler = _DummyOpenAIHandler()+ handler.config.optimize = True+ handler._compress_openai_responses_payload = MagicMock(+ return_value=(+ {"model": "gpt-5.4", "input": "hi"},+ False,+ 0,+ [],+ "router_no_compression",+ 10,+ 10,+ )+ )++ with patch.dict(sys.modules, {"websockets": fake_ws_mod}):+ await handler.handle_openai_responses_ws(client_ws)++ assert handler.compression_executor_calls == 1+ assert handler.compression_executor_timeouts == [COMPRESSION_TIMEOUT_SECONDS]+ handler._compress_openai_responses_payload.assert_called_once()++@pytest.mark.asyncioasync def test_happy_path_registry_empty_after_response_completed():"""Normal session completes — both relay tasks done, registry empty."""diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex 2c8f4fa6f..1c2b37c9f 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -1,13 +1,18 @@from __future__ import annotations+import asyncioimport base64import builtinsimport json+from types import SimpleNamespacefrom unittest.mock import patch+import httpx+from headroom.proxy.handlers.anthropic import AnthropicHandlerMixinfrom headroom.proxy.handlers.openai import OpenAIHandlerMixin, _decode_openai_bearer_payloadfrom headroom.proxy.helpers import _headroom_bypass_enabled+from headroom.proxy.server import HeadroomProxydef _jwt(payload: object) -> str:@@ -36,6 +41,32 @@ class _FreshCompressor:type(self).instances += 1+class _TimeoutHttpClient:+ async def request(self, **kwargs): # noqa: ANN001, ANN201+ raise httpx.ConnectTimeout("connect timed out")+++class _PassthroughRequest:+ method = "GET"+ headers = {}+ url = SimpleNamespace(path="/favicon.ico", query="")++ async def body(self) -> bytes:+ return b""+++class _RetryThenSuccessClient:+ def __init__(self) -> None:+ self.attempts = 0++ async def post(self, url, content, headers): # noqa: ANN001, ANN201+ self.attempts += 1+ if self.attempts == 1:+ raise httpx.ConnectTimeout("connect timed out")+ request = httpx.Request("POST", url, headers=headers, content=content)+ return httpx.Response(200, request=request, content=b"{}")++def test_decode_openai_bearer_payload_handles_missing_and_non_mapping_payloads() -> None:assert _decode_openai_bearer_payload({}) is Noneassert _decode_openai_bearer_payload({"authorization": "Basic abc"}) is None@@ -91,6 +122,47 @@ def test_headroom_bypass_helper_is_transport_neutral() -> None:assert OpenAIHandlerMixin._headroom_bypass_enabled({"x-headroom-bypass": "true"}) is True+def test_openai_passthrough_connect_timeout_returns_502() -> None:+ handler = object.__new__(OpenAIHandlerMixin)+ handler.http_client = _TimeoutHttpClient()++ async def run():+ return await handler.handle_passthrough(+ _PassthroughRequest(),+ "https://api.openai.com",+ )++ response = asyncio.run(run())++ assert response.status_code == 502+ payload = json.loads(response.body)+ assert payload["error"]["type"] == "connection_error"+ assert "Failed to connect to upstream API" in payload["error"]["message"]+++def test_retry_request_retries_connect_timeout() -> None:+ proxy = object.__new__(HeadroomProxy)+ proxy.http_client = _RetryThenSuccessClient()+ proxy.config = SimpleNamespace(+ retry_enabled=True,+ retry_max_attempts=2,+ retry_base_delay_ms=0,+ retry_max_delay_ms=0,+ )++ response = asyncio.run(+ proxy._retry_request(+ "POST",+ "https://api.openai.com/v1/responses",+ {},+ {"model": "gpt-5"},+ )+ )++ assert response.status_code == 200+ assert proxy.http_client.attempts == 2++def test_anthropic_tool_sort_and_context_append_helpers() -> None:tools = [{"type": "function", "function": {"name": "beta"}},