Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(proxy): let turn hooks rewrite OpenAI Responses requests on HTTP and WebSocket (#4232)

Tejas Choprafbf7bc3278c1
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 559fbc6ee..388205efe 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -22,6 +22,7 @@ from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import replace
from datetime import datetime
+from types import MappingProxyType
from typing import TYPE_CHECKING, Any
from urllib.parse import quote, unquote, urlparse
@@ -55,6 +56,7 @@ from headroom.proxy.tool_schema_savings_policy import (
net_hook_tool_saving,
without_deferral_flags,
)
+from headroom.proxy.turn_hooks import run_responses_create_hooks
from headroom.proxy.upstream_guard import is_safe_upstream_url
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
from headroom.proxy.ws_session_registry import (
@@ -1962,6 +1964,24 @@ class OpenAIHandlerMixin:
except Exception as exc:
logger.debug("[%s] Traffic learner (ws): %s", request_id, exc)
+ @staticmethod
+ async def _run_ws_create_hooks(
+ frame: Any, ws_headers: dict[str, str], connection_id: str
+ ) -> bool:
+ """Run ``on_responses_create`` on a WS create frame's inner payload.
+
+ True when a hook changed it, so the caller re-serializes the frame.
+ """
+ inner = frame.get("response", frame) if isinstance(frame, dict) else None
+ if not isinstance(inner, dict):
+ return False
+ return await run_responses_create_hooks(
+ inner,
+ headers=MappingProxyType(ws_headers),
+ transport="websocket",
+ connection_id=connection_id,
+ )
+
@staticmethod
def _headroom_bypass_enabled(headers: Any) -> bool:
"""Return True when inbound headers request full passthrough."""
@@ -5672,8 +5692,6 @@ class OpenAIHandlerMixin:
},
)
- model = body.get("model", "unknown")
- stream = body.get("stream", False)
body_mutation_tracker = BodyMutationTracker()
_bypass = self._headroom_bypass_enabled(request.headers)
if _bypass:
@@ -5693,8 +5711,17 @@ class OpenAIHandlerMixin:
url=str(request.url),
headers=dict(request.headers.items()),
body=body,
- metadata={"path": request.url.path, "stream": stream},
+ metadata={"path": request.url.path, "stream": body.get("stream", False)},
)
+ # Before every stage that reads the body, so a hook's replacement is
+ # what compression, deferral, token counting, pricing and the
+ # stream/buffered dispatch all see.
+ if not _bypass and await run_responses_create_hooks(
+ body, headers=request.headers, transport="http"
+ ):
+ body_mutation_tracker.mark_mutated("responses_create_hook")
+ model = body.get("model", "unknown")
+ stream = body.get("stream", False)
# /v1/responses uses provider-specific CompressionUnit extraction
# below, then routes mutable text through ContentRouter. The
@@ -7662,6 +7689,19 @@ class OpenAIHandlerMixin:
except json.JSONDecodeError:
# Not JSON — pass through as-is
pass
+ first_frame_is_create = isinstance(body, dict) and (
+ body.get("type") == "response.create" or ("type" not in body and "input" in body)
+ )
+ _ws_bypass = self._headroom_bypass_enabled(ws_headers)
+ if _ws_bypass:
+ logger.info(
+ "[%s] WS /v1/responses passthrough reason=bypass_header mutation=disabled",
+ request_id,
+ )
+ elif first_frame_is_create and await self._run_ws_create_hooks(
+ body, ws_headers, session_id
+ ):
+ first_msg_raw = json.dumps(body)
if isinstance(body, dict) and body:
ws_response_body_for_store = (
body["response"] if isinstance(body.get("response"), dict) else body
@@ -7707,12 +7747,6 @@ class OpenAIHandlerMixin:
ws_recorded_compression_timing_totals: dict[str, float] = {}
ws_ttfb_ms: float | None = None
ws_recorded_ttfb_ms = False
- _ws_bypass = self._headroom_bypass_enabled(ws_headers)
- if _ws_bypass:
- logger.info(
- "[%s] WS /v1/responses passthrough reason=bypass_header mutation=disabled",
- request_id,
- )
capture_codex_wire_debug(
"ws_inbound_first_frame",
@@ -8144,9 +8178,7 @@ class OpenAIHandlerMixin:
logger.warning(f"[{request_id}] WS Memory injection failed: {e}")
return frame_raw
- if isinstance(body, dict) and (
- body.get("type") == "response.create" or ("type" not in body and "input" in body)
- ):
+ if first_frame_is_create:
first_msg_raw = await _prepare_memory_frame(body, first_msg_raw)
first_response_body = body.get("response", body)
current_response_input = _responses_input_to_items(
@@ -8872,6 +8904,10 @@ class OpenAIHandlerMixin:
inbound_response = _inbound_frame_body.get(
"response", _inbound_frame_body
)
+ if not _ws_bypass and await self._run_ws_create_hooks(
+ _inbound_frame_body, ws_headers, session_id
+ ):
+ msg = json.dumps(_inbound_frame_body)
current_response_input = _responses_input_to_items(
inbound_response.get("input")
if isinstance(inbound_response, dict)
diff --git a/headroom/proxy/turn_hooks.py b/headroom/proxy/turn_hooks.py
index 5d9f6e228..e0dd0a886 100644
--- a/headroom/proxy/turn_hooks.py
+++ b/headroom/proxy/turn_hooks.py
@@ -21,10 +21,12 @@ the same deprecation policy.
from __future__ import annotations
+import inspect
import logging
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, field
-from typing import Any, Protocol, runtime_checkable
+from types import MappingProxyType
+from typing import Any, Literal, Protocol, runtime_checkable
from headroom.proxy.tool_schema_savings_policy import (
deferred_booking,
@@ -90,10 +92,31 @@ class TurnContext:
)
+@dataclass(frozen=True)
+class ResponsesCreate:
+ """One OpenAI Responses create request, before the proxy rewrites it.
+
+ ``body`` is a read-only view of the Responses payload (for a WebSocket
+ ``response.create`` frame, the inner ``response`` object). ``headers``
+ are the client's request headers, or the WebSocket handshake headers,
+ looked up by lowercase name. ``connection_id`` names the WebSocket
+ connection the frame arrived on (``None`` over HTTP): a
+ ``previous_response_id`` chain lives within one connection, so it is the
+ key for anything a hook must hold across a chain's frames. A reconnect
+ gets a new id; Codex's first frame on it replays the full transcript
+ rather than continuing the old chain, so it arrives as a fresh request.
+ """
+
+ body: Mapping[str, Any]
+ headers: Mapping[str, str]
+ transport: Literal["http", "websocket"]
+ connection_id: str | None = None
+
+
@runtime_checkable
class TurnHook(Protocol):
- """A registered turn observer. Both methods are optional (a hook may define
- either); missing methods are simply skipped."""
+ """A registered turn observer. Every method is optional (a hook may define
+ any of them); missing methods are simply skipped."""
name: str
@@ -104,9 +127,10 @@ class TurnHook(Protocol):
stream_safe: bool
# Optional: run order among registered hooks, lowest first (stable for
- # ties; absent ⇒ DEFAULT_HOOK_PRIORITY). Convention: routers that change
- # ``ctx.model`` 10-49, message folds 50-99, tool-catalog shaping that is
- # gated on the final model (deferral) 200+. Registration order is
+ # ties; absent ⇒ DEFAULT_HOOK_PRIORITY). Convention: routers 10-49 (an
+ # OpenAI Responses router rewrites the model in ``on_responses_create``,
+ # which runs before every ``on_request``), message folds 50-99,
+ # tool-catalog shaping that is gated on the final model (deferral) 200+. Registration order is
# entry-point discovery order, which an operator does not control, so a
# hook whose correctness depends on running after another must say so.
priority: int
@@ -122,6 +146,26 @@ class TurnHook(Protocol):
May ``await call_model(messages)`` to re-drive the model (e.g. to resolve
an injected tool call), looping as needed before returning the final."""
+ async def on_responses_create(self, request: ResponsesCreate) -> dict[str, Any] | None:
+ """Return a replacement body for an OpenAI Responses create request,
+ or ``None`` to leave it unchanged.
+
+ Called once per HTTP ``/v1/responses`` body and once per WebSocket
+ ``response.create`` frame, on the event loop, before compression and
+ every model-gated stage (not when the client sent the Headroom bypass
+ header). This is where a router changes the model or reasoning effort:
+ unlike ``on_request`` it runs whether or not compression is enabled.
+
+ Everything after it -- memory, compression, tool-search deferral,
+ token counting, pricing, metrics, the WS continuation template -- sees
+ the replacement; the next hook gets it as its ``request.body``. The
+ view is read-only at the top level and nested values are shared with
+ the live request, so build replacements (``{**body, "reasoning":
+ {...}}``) rather than mutating nested dicts or lists. A hook that
+ raises then leaves nothing half-applied, and a request no hook
+ replaced keeps the client's original bytes. Keep it fast: it is awaited inline on every frame, so
+ no network I/O."""
+
_hooks: list[TurnHook] = []
@@ -301,3 +345,43 @@ async def run_response_hooks(
if replacement is not None:
current = replacement
return current
+
+
+async def run_responses_create_hooks(
+ body: dict[str, Any],
+ *,
+ headers: Mapping[str, str],
+ transport: Literal["http", "websocket"],
+ connection_id: str | None = None,
+) -> bool:
+ """Run every hook's ``on_responses_create`` and apply the final body to
+ ``body`` in place. True if any hook replaced it.
+
+ Inert (returns False) when no hook defines the method. A failing hook is
+ logged and skipped. A sync method is accepted as well as a coroutine.
+ """
+ current: dict[str, Any] = body
+ for hook in registered_turn_hooks():
+ fn = getattr(hook, "on_responses_create", None)
+ if fn is None:
+ continue
+ request = ResponsesCreate(
+ body=MappingProxyType(current),
+ headers=headers,
+ transport=transport,
+ connection_id=connection_id,
+ )
+ try:
+ result = fn(request)
+ if inspect.isawaitable(result):
+ result = await result
+ except Exception: # a hook must never break the proxy
+ log.exception("turn hook %r on_responses_create failed", getattr(hook, "name", hook))
+ continue
+ if isinstance(result, dict):
+ current = result
+ if current is body:
+ return False
+ body.clear()
+ body.update(current)
+ return True
diff --git a/tests/test_responses_create_hooks.py b/tests/test_responses_create_hooks.py
new file mode 100644
index 000000000..9a275500d
--- /dev/null
+++ b/tests/test_responses_create_hooks.py
@@ -0,0 +1,296 @@
+"""``on_responses_create``: one routing seam for OpenAI Responses on both transports.
+
+Codex sends ``/v1/responses`` over a WebSocket when wrapped, and an HTTP
+middleware never sees those frames, so a router installed as middleware did
+nothing for wrapped Codex. A turn hook's ``on_responses_create`` runs on the
+HTTP body and on every WebSocket ``response.create`` frame, before the proxy's
+own stages, so a model/effort rewrite reaches the upstream and the accounting.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from headroom.proxy.turn_hooks import (
+ ResponsesCreate,
+ clear_turn_hooks,
+ register_turn_hook,
+)
+from tests.test_openai_codex_ws_lifecycle import (
+ _DummyOpenAIHandler,
+ _FakeUpstream,
+ _FakeWebSocket,
+ _make_fake_websockets_module,
+)
+
+
[email protected](autouse=True)
+def _clean_registry():
+ clear_turn_hooks()
+ yield
+ clear_turn_hooks()
+
+
+class _Router:
+ name = "test-router"
+
+ def __init__(self) -> None:
+ self.seen: list[tuple[str, str, str | None]] = []
+ self.connections: list[str | None] = []
+
+ async def on_responses_create(self, request: ResponsesCreate) -> dict:
+ self.seen.append(
+ (request.transport, str(request.body.get("model")), request.headers.get("user-agent"))
+ )
+ self.connections.append(request.connection_id)
+ return {**request.body, "model": "gpt-5.4-mini", "reasoning": {"effort": "low"}}
+
+
+def _create(text: str, **extra: Any) -> str:
+ return json.dumps(
+ {"type": "response.create", "response": {"model": "gpt-5.5", "input": text, **extra}}
+ )
+
+
+def _upstream() -> _FakeUpstream:
+ return _FakeUpstream(
+ [
+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
+ json.dumps({"type": "response.completed", "response": {"id": "r_1"}}),
+ ],
+ hold_after_events=True,
+ )
+
+
+async def _run_ws(frames: list[str], headers: dict[str, str] | None = None) -> list[dict]:
+ upstream = _upstream()
+ client_ws = _FakeWebSocket(frames=frames, headers=headers)
+ with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):
+ await _DummyOpenAIHandler().handle_openai_responses_ws(client_ws)
+ return [json.loads(frame) for frame in upstream.sent]
+
+
+async def test_ws_create_frames_carry_the_hooks_model_and_effort_upstream() -> None:
+ router = _Router()
+ register_turn_hook(router)
+
+ # previous_response_id chains are left to the client and OpenAI: a model
+ # switch mid-chain is allowed, so the hook's rewrite rides alongside it.
+ sent = await _run_ws(
+ [
+ _create("first"),
+ json.dumps({"type": "response.cancel"}),
+ _create("next", previous_response_id="r_1"),
+ ]
+ )
+
+ creates = [frame["response"] for frame in sent if frame.get("type") == "response.create"]
+ assert [c["model"] for c in creates] == ["gpt-5.4-mini", "gpt-5.4-mini"]
+ assert [c["reasoning"] for c in creates] == [{"effort": "low"}] * 2
+ assert creates[1]["previous_response_id"] == "r_1"
+ assert {"type": "response.cancel"} in sent
+ # Compression is off in the dummy handler: routing must not depend on it.
+ assert router.seen == [
+ ("websocket", "gpt-5.5", "codex-cli/0.5"),
+ ("websocket", "gpt-5.5", "codex-cli/0.5"),
+ ]
+ # One connection, one id: what a hook keys a previous_response_id chain on.
+ assert router.connections[0] and len(set(router.connections)) == 1
+
+
+async def test_ws_bypass_header_skips_the_hook() -> None:
+ router = _Router()
+ register_turn_hook(router)
+ frames = [_create("first"), _create("next")]
+
+ sent = await _run_ws(frames, headers={"authorization": "Bearer t", "x-headroom-bypass": "true"})
+
+ assert router.seen == []
+ assert [frame["response"]["model"] for frame in sent] == ["gpt-5.5", "gpt-5.5"]
+
+
+async def test_a_hook_that_raises_does_not_break_the_session() -> None:
+ class _Broken:
+ name = "broken"
+
+ async def on_responses_create(self, request: ResponsesCreate) -> None:
+ # The body is read-only: a hook cannot half-apply an edit and then fail.
+ request.body["model"] = "gpt-5.4-mini" # type: ignore[index]
+
+ register_turn_hook(_Broken())
+ frames = [_create("first")]
+
+ sent = await _run_ws(frames)
+
+ assert sent == [json.loads(frames[0])]
+
+
+class _NoOp:
+ name = "no-op"
+
+ async def on_responses_create(self, request: ResponsesCreate) -> None:
+ return None
+
+
+def _post_http(raw: bytes, headers: dict[str, str] | None = None) -> tuple[Any, list, list]:
+ """POST raw bytes to /v1/responses.
+
+ Returns the response, what reached the upstream call as
+ ``(body, body_mutated)`` pairs, and the recorded outcomes.
+ """
+ pytest.importorskip("fastapi")
+ httpx = pytest.importorskip("httpx")
+ from fastapi.testclient import TestClient
+
+ from headroom.proxy.loopback_guard import require_loopback
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ sent: list[Any] = []
+ outcomes: list[Any] = []
+
+ with TestClient(app) as client:
+ server = app.state.proxy
+
+ async def fake_retry(method, url, headers, body, stream=False, **kwargs):
+ sent.append((_as_json(body), kwargs.get("body_mutated")))
+ return httpx.Response(
+ 200,
+ json={
+ "id": "resp_1",
+ "object": "response",
+ "status": "completed",
+ "model": "gpt-5.4-mini",
+ "output": [],
+ "usage": {"input_tokens": 10, "output_tokens": 2},
+ },
+ request=httpx.Request("POST", url),
+ )
+
+ record = server._record_request_outcome
+
+ async def capture(outcome):
+ outcomes.append(outcome)
+ await record(outcome)
+
+ server._retry_request = fake_retry
+ server._record_request_outcome = capture
+ resp = client.post(
+ "/v1/responses",
+ content=raw,
+ headers={
+ "Authorization": "Bearer sk-test",
+ "Content-Type": "application/json",
+ "User-Agent": "codex_exec/0.162.1",
+ **(headers or {}),
+ },
+ )
+ return resp, sent, outcomes
+
+
+def _as_json(body: Any) -> dict:
+ return json.loads(body) if isinstance(body, (bytes, str)) else body
+
+
+_RAW = json.dumps({"model": "gpt-5.5", "input": "hi", "stream": False}).encode()
+
+
+def test_http_body_carries_the_hooks_model_and_the_outcome_prices_it() -> None:
+ router = _Router()
+ register_turn_hook(router)
+
+ resp, sent, outcomes = _post_http(_RAW)
+
+ assert resp.status_code == 200, resp.text
+ assert router.seen == [("http", "gpt-5.5", "codex_exec/0.162.1")]
+ assert router.connections == [None]
+ body, mutated = sent[0]
+ assert body["model"] == "gpt-5.4-mini"
+ assert body["reasoning"] == {"effort": "low"}
+ assert mutated is True
+ assert outcomes and all(o.model == "gpt-5.4-mini" for o in outcomes)
+
+
+def test_http_hook_that_changes_nothing_keeps_the_clients_bytes() -> None:
+ register_turn_hook(_NoOp())
+
+ resp, sent, _ = _post_http(_RAW)
+
+ assert resp.status_code == 200, resp.text
+ # Unmutated is what sends the client's original bytes (#1542).
+ assert sent == [(json.loads(_RAW), False)]
+
+
+def test_http_bypass_header_skips_the_hook() -> None:
+ router = _Router()
+ register_turn_hook(router)
+
+ resp, sent, _ = _post_http(_RAW, headers={"x-headroom-bypass": "true"})
+
+ assert resp.status_code == 200, resp.text
+ assert router.seen == []
+ assert sent == [(json.loads(_RAW), False)]
+
+
+def test_http_hook_that_turns_on_streaming_takes_the_streaming_path() -> None:
+ class _Streamer:
+ name = "streamer"
+
+ async def on_responses_create(self, request: ResponsesCreate) -> dict:
+ return {**request.body, "stream": True}
+
+ register_turn_hook(_Streamer())
+ pytest.importorskip("fastapi")
+ from fastapi.responses import JSONResponse
+ from fastapi.testclient import TestClient
+
+ from headroom.proxy.loopback_guard import require_loopback
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ streamed: list[dict] = []
+
+ with TestClient(app) as client:
+ server = app.state.proxy
+
+ async def fake_stream(url, headers, body, *args, **kwargs):
+ streamed.append(body)
+ return JSONResponse({"ok": True})
+
+ async def no_buffered(*args, **kwargs):
+ raise AssertionError("a streaming upstream request took the buffered path")
+
+ server._stream_response = fake_stream
+ server._retry_request = no_buffered
+ resp = client.post(
+ "/v1/responses",
+ content=_RAW,
+ headers={"Authorization": "Bearer sk-test", "Content-Type": "application/json"},
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert [b["stream"] for b in streamed] == [True]