Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(proxy): let turn hooks rewrite OpenAI Responses requests on HTTP and WebSocket (#4232)
Tejas Choprafbf7bc3278c1
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 559fbc6ee..388205efe 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -22,6 +22,7 @@ from collections import OrderedDictfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom dataclasses import replacefrom datetime import datetime+from types import MappingProxyTypefrom typing import TYPE_CHECKING, Anyfrom urllib.parse import quote, unquote, urlparse@@ -55,6 +56,7 @@ from headroom.proxy.tool_schema_savings_policy import (net_hook_tool_saving,without_deferral_flags,)+from headroom.proxy.turn_hooks import run_responses_create_hooksfrom headroom.proxy.upstream_guard import is_safe_upstream_urlfrom headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERSfrom headroom.proxy.ws_session_registry import (@@ -1962,6 +1964,24 @@ class OpenAIHandlerMixin:except Exception as exc:logger.debug("[%s] Traffic learner (ws): %s", request_id, exc)+ @staticmethod+ async def _run_ws_create_hooks(+ frame: Any, ws_headers: dict[str, str], connection_id: str+ ) -> bool:+ """Run ``on_responses_create`` on a WS create frame's inner payload.++ True when a hook changed it, so the caller re-serializes the frame.+ """+ inner = frame.get("response", frame) if isinstance(frame, dict) else None+ if not isinstance(inner, dict):+ return False+ return await run_responses_create_hooks(+ inner,+ headers=MappingProxyType(ws_headers),+ transport="websocket",+ connection_id=connection_id,+ )+@staticmethoddef _headroom_bypass_enabled(headers: Any) -> bool:"""Return True when inbound headers request full passthrough."""@@ -5672,8 +5692,6 @@ class OpenAIHandlerMixin:},)- model = body.get("model", "unknown")- stream = body.get("stream", False)body_mutation_tracker = BodyMutationTracker()_bypass = self._headroom_bypass_enabled(request.headers)if _bypass:@@ -5693,8 +5711,17 @@ class OpenAIHandlerMixin:url=str(request.url),headers=dict(request.headers.items()),body=body,- metadata={"path": request.url.path, "stream": stream},+ metadata={"path": request.url.path, "stream": body.get("stream", False)},)+ # Before every stage that reads the body, so a hook's replacement is+ # what compression, deferral, token counting, pricing and the+ # stream/buffered dispatch all see.+ if not _bypass and await run_responses_create_hooks(+ body, headers=request.headers, transport="http"+ ):+ body_mutation_tracker.mark_mutated("responses_create_hook")+ model = body.get("model", "unknown")+ stream = body.get("stream", False)# /v1/responses uses provider-specific CompressionUnit extraction# below, then routes mutable text through ContentRouter. The@@ -7662,6 +7689,19 @@ class OpenAIHandlerMixin:except json.JSONDecodeError:# Not JSON — pass through as-ispass+ first_frame_is_create = isinstance(body, dict) and (+ body.get("type") == "response.create" or ("type" not in body and "input" in body)+ )+ _ws_bypass = self._headroom_bypass_enabled(ws_headers)+ if _ws_bypass:+ logger.info(+ "[%s] WS /v1/responses passthrough reason=bypass_header mutation=disabled",+ request_id,+ )+ elif first_frame_is_create and await self._run_ws_create_hooks(+ body, ws_headers, session_id+ ):+ first_msg_raw = json.dumps(body)if isinstance(body, dict) and body:ws_response_body_for_store = (body["response"] if isinstance(body.get("response"), dict) else body@@ -7707,12 +7747,6 @@ class OpenAIHandlerMixin:ws_recorded_compression_timing_totals: dict[str, float] = {}ws_ttfb_ms: float | None = Nonews_recorded_ttfb_ms = False- _ws_bypass = self._headroom_bypass_enabled(ws_headers)- if _ws_bypass:- logger.info(- "[%s] WS /v1/responses passthrough reason=bypass_header mutation=disabled",- request_id,- )capture_codex_wire_debug("ws_inbound_first_frame",@@ -8144,9 +8178,7 @@ class OpenAIHandlerMixin:logger.warning(f"[{request_id}] WS Memory injection failed: {e}")return frame_raw- if isinstance(body, dict) and (- body.get("type") == "response.create" or ("type" not in body and "input" in body)- ):+ if first_frame_is_create:first_msg_raw = await _prepare_memory_frame(body, first_msg_raw)first_response_body = body.get("response", body)current_response_input = _responses_input_to_items(@@ -8872,6 +8904,10 @@ class OpenAIHandlerMixin:inbound_response = _inbound_frame_body.get("response", _inbound_frame_body)+ if not _ws_bypass and await self._run_ws_create_hooks(+ _inbound_frame_body, ws_headers, session_id+ ):+ msg = json.dumps(_inbound_frame_body)current_response_input = _responses_input_to_items(inbound_response.get("input")if isinstance(inbound_response, dict)diff --git a/headroom/proxy/turn_hooks.py b/headroom/proxy/turn_hooks.pyindex 5d9f6e228..e0dd0a886 100644--- a/headroom/proxy/turn_hooks.py+++ b/headroom/proxy/turn_hooks.py@@ -21,10 +21,12 @@ the same deprecation policy.from __future__ import annotations+import inspectimport logging-from collections.abc import Awaitable, Callable+from collections.abc import Awaitable, Callable, Mappingfrom dataclasses import dataclass, field-from typing import Any, Protocol, runtime_checkable+from types import MappingProxyType+from typing import Any, Literal, Protocol, runtime_checkablefrom headroom.proxy.tool_schema_savings_policy import (deferred_booking,@@ -90,10 +92,31 @@ class TurnContext:)+@dataclass(frozen=True)+class ResponsesCreate:+ """One OpenAI Responses create request, before the proxy rewrites it.++ ``body`` is a read-only view of the Responses payload (for a WebSocket+ ``response.create`` frame, the inner ``response`` object). ``headers``+ are the client's request headers, or the WebSocket handshake headers,+ looked up by lowercase name. ``connection_id`` names the WebSocket+ connection the frame arrived on (``None`` over HTTP): a+ ``previous_response_id`` chain lives within one connection, so it is the+ key for anything a hook must hold across a chain's frames. A reconnect+ gets a new id; Codex's first frame on it replays the full transcript+ rather than continuing the old chain, so it arrives as a fresh request.+ """++ body: Mapping[str, Any]+ headers: Mapping[str, str]+ transport: Literal["http", "websocket"]+ connection_id: str | None = None++@runtime_checkableclass TurnHook(Protocol):- """A registered turn observer. Both methods are optional (a hook may define- either); missing methods are simply skipped."""+ """A registered turn observer. Every method is optional (a hook may define+ any of them); missing methods are simply skipped."""name: str@@ -104,9 +127,10 @@ class TurnHook(Protocol):stream_safe: bool# Optional: run order among registered hooks, lowest first (stable for- # ties; absent ⇒ DEFAULT_HOOK_PRIORITY). Convention: routers that change- # ``ctx.model`` 10-49, message folds 50-99, tool-catalog shaping that is- # gated on the final model (deferral) 200+. Registration order is+ # ties; absent ⇒ DEFAULT_HOOK_PRIORITY). Convention: routers 10-49 (an+ # OpenAI Responses router rewrites the model in ``on_responses_create``,+ # which runs before every ``on_request``), message folds 50-99,+ # tool-catalog shaping that is gated on the final model (deferral) 200+. Registration order is# entry-point discovery order, which an operator does not control, so a# hook whose correctness depends on running after another must say so.priority: int@@ -122,6 +146,26 @@ class TurnHook(Protocol):May ``await call_model(messages)`` to re-drive the model (e.g. to resolvean injected tool call), looping as needed before returning the final."""+ async def on_responses_create(self, request: ResponsesCreate) -> dict[str, Any] | None:+ """Return a replacement body for an OpenAI Responses create request,+ or ``None`` to leave it unchanged.++ Called once per HTTP ``/v1/responses`` body and once per WebSocket+ ``response.create`` frame, on the event loop, before compression and+ every model-gated stage (not when the client sent the Headroom bypass+ header). This is where a router changes the model or reasoning effort:+ unlike ``on_request`` it runs whether or not compression is enabled.++ Everything after it -- memory, compression, tool-search deferral,+ token counting, pricing, metrics, the WS continuation template -- sees+ the replacement; the next hook gets it as its ``request.body``. The+ view is read-only at the top level and nested values are shared with+ the live request, so build replacements (``{**body, "reasoning":+ {...}}``) rather than mutating nested dicts or lists. A hook that+ raises then leaves nothing half-applied, and a request no hook+ replaced keeps the client's original bytes. Keep it fast: it is awaited inline on every frame, so+ no network I/O."""+_hooks: list[TurnHook] = []@@ -301,3 +345,43 @@ async def run_response_hooks(if replacement is not None:current = replacementreturn current+++async def run_responses_create_hooks(+ body: dict[str, Any],+ *,+ headers: Mapping[str, str],+ transport: Literal["http", "websocket"],+ connection_id: str | None = None,+) -> bool:+ """Run every hook's ``on_responses_create`` and apply the final body to+ ``body`` in place. True if any hook replaced it.++ Inert (returns False) when no hook defines the method. A failing hook is+ logged and skipped. A sync method is accepted as well as a coroutine.+ """+ current: dict[str, Any] = body+ for hook in registered_turn_hooks():+ fn = getattr(hook, "on_responses_create", None)+ if fn is None:+ continue+ request = ResponsesCreate(+ body=MappingProxyType(current),+ headers=headers,+ transport=transport,+ connection_id=connection_id,+ )+ try:+ result = fn(request)+ if inspect.isawaitable(result):+ result = await result+ except Exception: # a hook must never break the proxy+ log.exception("turn hook %r on_responses_create failed", getattr(hook, "name", hook))+ continue+ if isinstance(result, dict):+ current = result+ if current is body:+ return False+ body.clear()+ body.update(current)+ return Truediff --git a/tests/test_responses_create_hooks.py b/tests/test_responses_create_hooks.pynew file mode 100644index 000000000..9a275500d--- /dev/null+++ b/tests/test_responses_create_hooks.py@@ -0,0 +1,296 @@+"""``on_responses_create``: one routing seam for OpenAI Responses on both transports.++Codex sends ``/v1/responses`` over a WebSocket when wrapped, and an HTTP+middleware never sees those frames, so a router installed as middleware did+nothing for wrapped Codex. A turn hook's ``on_responses_create`` runs on the+HTTP body and on every WebSocket ``response.create`` frame, before the proxy's+own stages, so a model/effort rewrite reaches the upstream and the accounting.+"""++from __future__ import annotations++import json+import sys+from typing import Any+from unittest.mock import patch++import pytest++from headroom.proxy.turn_hooks import (+ ResponsesCreate,+ clear_turn_hooks,+ register_turn_hook,+)+from tests.test_openai_codex_ws_lifecycle import (+ _DummyOpenAIHandler,+ _FakeUpstream,+ _FakeWebSocket,+ _make_fake_websockets_module,+)++[email protected](autouse=True)+def _clean_registry():+ clear_turn_hooks()+ yield+ clear_turn_hooks()+++class _Router:+ name = "test-router"++ def __init__(self) -> None:+ self.seen: list[tuple[str, str, str | None]] = []+ self.connections: list[str | None] = []++ async def on_responses_create(self, request: ResponsesCreate) -> dict:+ self.seen.append(+ (request.transport, str(request.body.get("model")), request.headers.get("user-agent"))+ )+ self.connections.append(request.connection_id)+ return {**request.body, "model": "gpt-5.4-mini", "reasoning": {"effort": "low"}}+++def _create(text: str, **extra: Any) -> str:+ return json.dumps(+ {"type": "response.create", "response": {"model": "gpt-5.5", "input": text, **extra}}+ )+++def _upstream() -> _FakeUpstream:+ return _FakeUpstream(+ [+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),+ json.dumps({"type": "response.completed", "response": {"id": "r_1"}}),+ ],+ hold_after_events=True,+ )+++async def _run_ws(frames: list[str], headers: dict[str, str] | None = None) -> list[dict]:+ upstream = _upstream()+ client_ws = _FakeWebSocket(frames=frames, headers=headers)+ with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):+ await _DummyOpenAIHandler().handle_openai_responses_ws(client_ws)+ return [json.loads(frame) for frame in upstream.sent]+++async def test_ws_create_frames_carry_the_hooks_model_and_effort_upstream() -> None:+ router = _Router()+ register_turn_hook(router)++ # previous_response_id chains are left to the client and OpenAI: a model+ # switch mid-chain is allowed, so the hook's rewrite rides alongside it.+ sent = await _run_ws(+ [+ _create("first"),+ json.dumps({"type": "response.cancel"}),+ _create("next", previous_response_id="r_1"),+ ]+ )++ creates = [frame["response"] for frame in sent if frame.get("type") == "response.create"]+ assert [c["model"] for c in creates] == ["gpt-5.4-mini", "gpt-5.4-mini"]+ assert [c["reasoning"] for c in creates] == [{"effort": "low"}] * 2+ assert creates[1]["previous_response_id"] == "r_1"+ assert {"type": "response.cancel"} in sent+ # Compression is off in the dummy handler: routing must not depend on it.+ assert router.seen == [+ ("websocket", "gpt-5.5", "codex-cli/0.5"),+ ("websocket", "gpt-5.5", "codex-cli/0.5"),+ ]+ # One connection, one id: what a hook keys a previous_response_id chain on.+ assert router.connections[0] and len(set(router.connections)) == 1+++async def test_ws_bypass_header_skips_the_hook() -> None:+ router = _Router()+ register_turn_hook(router)+ frames = [_create("first"), _create("next")]++ sent = await _run_ws(frames, headers={"authorization": "Bearer t", "x-headroom-bypass": "true"})++ assert router.seen == []+ assert [frame["response"]["model"] for frame in sent] == ["gpt-5.5", "gpt-5.5"]+++async def test_a_hook_that_raises_does_not_break_the_session() -> None:+ class _Broken:+ name = "broken"++ async def on_responses_create(self, request: ResponsesCreate) -> None:+ # The body is read-only: a hook cannot half-apply an edit and then fail.+ request.body["model"] = "gpt-5.4-mini" # type: ignore[index]++ register_turn_hook(_Broken())+ frames = [_create("first")]++ sent = await _run_ws(frames)++ assert sent == [json.loads(frames[0])]+++class _NoOp:+ name = "no-op"++ async def on_responses_create(self, request: ResponsesCreate) -> None:+ return None+++def _post_http(raw: bytes, headers: dict[str, str] | None = None) -> tuple[Any, list, list]:+ """POST raw bytes to /v1/responses.++ Returns the response, what reached the upstream call as+ ``(body, body_mutated)`` pairs, and the recorded outcomes.+ """+ pytest.importorskip("fastapi")+ httpx = pytest.importorskip("httpx")+ from fastapi.testclient import TestClient++ from headroom.proxy.loopback_guard import require_loopback+ from headroom.proxy.server import ProxyConfig, create_app++ app = create_app(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ )+ app.dependency_overrides[require_loopback] = lambda: None+ sent: list[Any] = []+ outcomes: list[Any] = []++ with TestClient(app) as client:+ server = app.state.proxy++ async def fake_retry(method, url, headers, body, stream=False, **kwargs):+ sent.append((_as_json(body), kwargs.get("body_mutated")))+ return httpx.Response(+ 200,+ json={+ "id": "resp_1",+ "object": "response",+ "status": "completed",+ "model": "gpt-5.4-mini",+ "output": [],+ "usage": {"input_tokens": 10, "output_tokens": 2},+ },+ request=httpx.Request("POST", url),+ )++ record = server._record_request_outcome++ async def capture(outcome):+ outcomes.append(outcome)+ await record(outcome)++ server._retry_request = fake_retry+ server._record_request_outcome = capture+ resp = client.post(+ "/v1/responses",+ content=raw,+ headers={+ "Authorization": "Bearer sk-test",+ "Content-Type": "application/json",+ "User-Agent": "codex_exec/0.162.1",+ **(headers or {}),+ },+ )+ return resp, sent, outcomes+++def _as_json(body: Any) -> dict:+ return json.loads(body) if isinstance(body, (bytes, str)) else body+++_RAW = json.dumps({"model": "gpt-5.5", "input": "hi", "stream": False}).encode()+++def test_http_body_carries_the_hooks_model_and_the_outcome_prices_it() -> None:+ router = _Router()+ register_turn_hook(router)++ resp, sent, outcomes = _post_http(_RAW)++ assert resp.status_code == 200, resp.text+ assert router.seen == [("http", "gpt-5.5", "codex_exec/0.162.1")]+ assert router.connections == [None]+ body, mutated = sent[0]+ assert body["model"] == "gpt-5.4-mini"+ assert body["reasoning"] == {"effort": "low"}+ assert mutated is True+ assert outcomes and all(o.model == "gpt-5.4-mini" for o in outcomes)+++def test_http_hook_that_changes_nothing_keeps_the_clients_bytes() -> None:+ register_turn_hook(_NoOp())++ resp, sent, _ = _post_http(_RAW)++ assert resp.status_code == 200, resp.text+ # Unmutated is what sends the client's original bytes (#1542).+ assert sent == [(json.loads(_RAW), False)]+++def test_http_bypass_header_skips_the_hook() -> None:+ router = _Router()+ register_turn_hook(router)++ resp, sent, _ = _post_http(_RAW, headers={"x-headroom-bypass": "true"})++ assert resp.status_code == 200, resp.text+ assert router.seen == []+ assert sent == [(json.loads(_RAW), False)]+++def test_http_hook_that_turns_on_streaming_takes_the_streaming_path() -> None:+ class _Streamer:+ name = "streamer"++ async def on_responses_create(self, request: ResponsesCreate) -> dict:+ return {**request.body, "stream": True}++ register_turn_hook(_Streamer())+ pytest.importorskip("fastapi")+ from fastapi.responses import JSONResponse+ from fastapi.testclient import TestClient++ from headroom.proxy.loopback_guard import require_loopback+ from headroom.proxy.server import ProxyConfig, create_app++ app = create_app(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ )+ app.dependency_overrides[require_loopback] = lambda: None+ streamed: list[dict] = []++ with TestClient(app) as client:+ server = app.state.proxy++ async def fake_stream(url, headers, body, *args, **kwargs):+ streamed.append(body)+ return JSONResponse({"ok": True})++ async def no_buffered(*args, **kwargs):+ raise AssertionError("a streaming upstream request took the buffered path")++ server._stream_response = fake_stream+ server._retry_request = no_buffered+ resp = client.post(+ "/v1/responses",+ content=_RAW,+ headers={"Authorization": "Bearer sk-test", "Content-Type": "application/json"},+ )++ assert resp.status_code == 200, resp.text+ assert [b["stream"] for b in streamed] == [True]