Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Tejas/turn hooks extension (#1903)
Tejas Choprac9217856d3c2
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.htmlindex f5aa65d3c..54d04c240 100644--- a/headroom/dashboard/templates/dashboard.html+++ b/headroom/dashboard/templates/dashboard.html@@ -269,6 +269,20 @@</template></div>+ <!-- Tool-Schema Deferral (tool search) — only rendered when there's a saving to show -->+ <template x-if="(stats.savings?.by_layer?.tool_search?.tokens || 0) > 0">+ <div class="bg-surface rounded-lg p-4 border border-border">+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tool-Schema Deferral</div>+ <div class="flex items-baseline gap-2">+ <span class="text-3xl font-light tabular-nums text-emerald-400" x-text="formatNumber(stats.savings?.by_layer?.tool_search?.tokens || 0)"></span>+ <span class="text-sm text-gray-400">tokens</span>+ </div>+ <div class="mt-1 text-xs text-gray-600 leading-relaxed"+ x-text="formatNumber(stats.savings?.by_layer?.tool_search?.requests || 0) + ' calls · tool schemas deferred (recent window)'">+ </div>+ </div>+ </template>+<!-- Headroom Overhead --><div class="bg-surface rounded-lg p-4 border border-border"><div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Overhead</div>@@ -2002,6 +2016,7 @@<span class="text-xs text-gray-600 shrink-0">·</span><span class="text-xs text-emerald-400 shrink-0">${tokensSaved} tok</span><span class="text-xs text-gray-600 shrink-0">(${savingsPct}%)</span>+ ${(t.tool_schema_saved_tokens || 0) > 0 ? `<span class="text-xs text-cyan-400 shrink-0" title="Tool-definition tokens deferred out of context">+${t.tool_schema_saved_tokens} tool-schema</span>` : ''}</div><span class="text-xs text-gray-600 shrink-0">${time}</span></div>diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 3e5ff61f4..86a12ce43 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -2758,6 +2758,52 @@ class OpenAIHandlerMixin:optimized_tokens = tokenizer.count_messages(body["messages"])tokens_saved = original_tokens - optimized_tokens+ # Turn hooks (opt-in extensions): a registered hook may rewrite the+ # outbound tools/messages before we send. Buffered requests only — a+ # streamed turn can't be re-driven to resolve whatever the model asks to+ # load. Gated on the registry so it is a no-op when none are registered;+ # the net tool-schema token delta is recorded so it shows up as a saving.+ from headroom.proxy.turn_hooks import (+ TurnContext,+ registered_turn_hooks,+ run_request_hooks,+ )++ if registered_turn_hooks() and not stream:+ _th_tools_before = body.get("tools")+ _th_tok_before = (+ tokenizer.count_text(json.dumps(_th_tools_before, default=str))+ if _th_tools_before+ else 0+ )+ _th_ctx = TurnContext(+ provider="openai",+ model=str(model),+ messages=body["messages"],+ tools=_th_tools_before,+ config=self.config,+ )+ run_request_hooks(_th_ctx)+ # A hook may either replace ctx.messages/ctx.tools or mutate them in+ # place (the contract allows both). Use object identity only to decide+ # whether body needs reassignment; measure the saving from the FINAL+ # tools object regardless, so an in-place shrink is still counted.+ if _th_ctx.messages is not body["messages"]:+ optimized_messages = _th_ctx.messages+ body["messages"] = optimized_messages+ if _th_ctx.tools is not _th_tools_before:+ tools = _th_ctx.tools+ body["tools"] = tools+ _th_tok_after = (+ tokenizer.count_text(json.dumps(_th_ctx.tools, default=str)) if _th_ctx.tools else 0+ )+ _th_saved = max(0, _th_tok_before - _th_tok_after)+ if _th_saved > 0:+ tags["turn_hook_tools_saved_tokens"] = (+ int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved+ )+ transforms_applied.append(f"turn_hook:tools:{_th_saved}tok")+# Compatibility shim: GPT-5 / o-series chat models REJECT the legacy# `max_tokens` ("Unsupported parameter … Use 'max_completion_tokens'# instead"); gpt-4o/4.1 accept `max_completion_tokens` too. openai-@@ -3071,6 +3117,54 @@ class OpenAIHandlerMixin:else:headers = await apply_copilot_api_auth(headers, url=url)response = await self._retry_request("POST", url, headers, body)++ # Turn hooks: a registered extension may re-drive this turn+ # (e.g. resolve a tool the model asked to load) before we treat+ # the response as final. Buffered path only; no-op when no hook+ # is registered.+ from headroom.proxy.turn_hooks import (+ TurnContext as _TurnContext,+ )+ from headroom.proxy.turn_hooks import (+ registered_turn_hooks as _registered_turn_hooks,+ )+ from headroom.proxy.turn_hooks import (+ run_response_hooks,+ )++ if _registered_turn_hooks() and response.status_code == 200:+ try:+ _hook_resp_json = response.json()+ except (ValueError, json.JSONDecodeError):+ _hook_resp_json = None+ if isinstance(_hook_resp_json, dict):+ _hook_ctx = _TurnContext(+ provider="openai",+ model=str(model),+ messages=body["messages"],+ tools=body.get("tools"),+ config=self.config,+ )++ async def _hook_call_model(_msgs):+ body["messages"] = _msgs+ _r = await self._retry_request("POST", url, headers, body)+ return _r.json()++ _hook_final = await run_response_hooks(+ _hook_ctx, _hook_resp_json, _hook_call_model+ )+ if _hook_final is not _hook_resp_json:+ response = httpx.Response(+ status_code=200,+ headers={+ k: v+ for k, v in response.headers.items()+ if k.lower() not in ("content-encoding", "content-length")+ },+ content=json.dumps(_hook_final).encode(),+ )+self.pipeline_extensions.emit(PipelineStage.POST_SEND,operation="proxy.request",diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 1759627b0..3d69e5ff6 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -2012,6 +2012,24 @@ def _is_known_websocket_callback_failure(context: dict[str, Any]) -> bool:)+def _tool_schema_saved_from_tags(tags: object) -> int:+ """Tool-definition tokens Headroom kept out of context for one request by+ deferring heavy tool schemas: the native tool-search injection+ (``tool_search_deferred_tokens``) plus any registered turn-hook tools+ rewrite (``turn_hook_tools_saved_tokens``). Both tags are set only on the+ path where Headroom performed the deferral, so a client that already had+ tool search enabled (e.g. Claude Code / Codex) contributes zero here."""+ if not isinstance(tags, dict):+ return 0+ total = 0+ for key in ("tool_search_deferred_tokens", "turn_hook_tools_saved_tokens"):+ try:+ total += int(tags.get(key, 0) or 0)+ except (TypeError, ValueError):+ continue+ return total++def create_app(config: ProxyConfig | None = None) -> FastAPI:"""Create FastAPI application."""if not FASTAPI_AVAILABLE:@@ -2945,6 +2963,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"total_latency_ms": log.get("total_latency_ms"),"transforms_applied": log.get("transforms_applied", []),"waste_signals": log.get("waste_signals"),+ "tool_schema_saved_tokens": _tool_schema_saved_from_tags(log.get("tags")),}for log in recent_request_logsif log.get("input_tokens_original") is not None@@ -3129,6 +3148,19 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:display_session = persistent_savings.get("display_session", {})recent_request_logs = proxy.logger.get_recent(10_000) if proxy.logger else []recent_request_payload = _build_recent_request_payload()++ # Tool-schema deferral savings: tool-definition tokens kept out of the+ # model's context by deferring heavy schemas until they're needed+ # (native tool-search injection + any registered turn-hook tools+ # rewrite). Attributed to Headroom only — see _tool_schema_saved_from_tags.+ # Aggregated over the recent request-log window.+ tool_schema_tokens = 0+ tool_schema_requests = 0+ for _ts_log in recent_request_logs:+ _ts_saved = _tool_schema_saved_from_tags(_ts_log.get("tags"))+ if _ts_saved > 0:+ tool_schema_tokens += _ts_saved+ tool_schema_requests += 1agent_usage = _build_agent_usage_summary(recent_request_logs,requests_by_provider=dict(m.requests_by_provider),@@ -3229,6 +3261,20 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"(A/B holdout), always with a confidence band."),},+ "tool_search": {+ "tokens": tool_schema_tokens,+ "tokens_saved": tool_schema_tokens,+ "requests": tool_schema_requests,+ "window": len(recent_request_logs),+ "description": (+ "Tool-definition tokens kept out of the model's context "+ "by deferring heavy tool schemas until they're searched "+ "for. Counted only when Headroom performed the deferral — "+ "not when the client (e.g. Claude Code / Codex) already "+ "had tool search enabled. Aggregated over the recent "+ "request window."+ ),+ },},},"requests": {diff --git a/tests/test_openai_chat_turn_hooks.py b/tests/test_openai_chat_turn_hooks.pynew file mode 100644index 000000000..15f6b27d0--- /dev/null+++ b/tests/test_openai_chat_turn_hooks.py@@ -0,0 +1,295 @@+"""End-to-end turn-hook wiring on the OpenAI chat-completions direct path.++Proves the two seams added to ``handle_openai_chat`` for the direct+(no-backend) buffered path:++* ``on_request`` fires before the upstream send — a hook can shrink the+ outbound ``tools``, and the net tool-schema token delta is recorded as a+ saving (surfaced via the ``x-headroom-transforms`` header / tags).+* ``on_response`` fires after the send with a working ``call_model`` — a hook+ can detect a tool the model asked to load, re-drive the model, and have the+ proxy return the *final* response transparently.++Uses a fake hook (mimicking the tool-router extension's shrink + reload) and a+mocked ``_retry_request`` so no network / real provider is needed. Also pins the+no-op property: with no hook registered the path is unchanged.+"""++from __future__ import annotations++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402+from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook # noqa: E402++_SEARCH_TOOL = "search_tools"++[email protected](autouse=True)+def _clean_hooks():+ clear_turn_hooks()+ yield+ clear_turn_hooks()+++def _big_tool(name: str) -> dict:+ return {+ "type": "function",+ "function": {+ "name": name,+ "description": f"{name} does a thing " + ("x " * 40),+ "parameters": {+ "type": "object",+ "properties": {"arg": {"type": "string", "description": "y " * 60}},+ },+ },+ }+++def _tools(n: int = 13) -> list[dict]:+ return [_big_tool(f"tool_{i}") for i in range(n)]+++def _search_call_response() -> dict:+ return {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {+ "name": _SEARCH_TOOL,+ "arguments": '{"query":"do a thing"}',+ },+ }+ ],+ },+ "finish_reason": "tool_calls",+ }+ ],+ "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},+ }+++def _final_response() -> dict:+ return {+ "id": "chatcmpl-2",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "all done"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 120, "completion_tokens": 5, "total_tokens": 125},+ }+++class _FakeRouterHook:+ """Mimics the tool-router extension: shrink on request, reload on response."""++ name = "fake_router"++ def __init__(self):+ self.on_request_calls = 0+ self.on_response_calls = 0++ def on_request(self, ctx):+ self.on_request_calls += 1+ # Shrink: drop all but the first tool + inject a search_tools stub.+ if isinstance(ctx.tools, list) and len(ctx.tools) > 2:+ ctx.tools = [ctx.tools[0], {"type": "function", "function": {"name": _SEARCH_TOOL}}]++ async def on_response(self, ctx, response, call_model):+ self.on_response_calls += 1+ tcs = (response.get("choices") or [{}])[0].get("message", {}).get("tool_calls") or []+ if any(tc.get("function", {}).get("name") == _SEARCH_TOOL for tc in tcs):+ return await call_model(ctx.messages + [{"role": "user", "content": "resolved"}])+ return None+++def _config() -> ProxyConfig:+ # No backend -> the "Direct OpenAI API (no backend configured)" path.+ return ProxyConfig(optimize=False, cache_enabled=False, rate_limit_enabled=False)+++def _post(client: TestClient, body: dict):+ return client.post(+ "/v1/chat/completions",+ json=body,+ headers={"Authorization": "Bearer test-key"},+ )+++def test_direct_path_shrinks_then_reloads_and_returns_final():+ hook = _FakeRouterHook()+ register_turn_hook(hook)++ seen_bodies: list[dict] = []++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ # capture the exact outbound body per upstream call+ import copy++ seen_bodies.append(copy.deepcopy(body))+ payload = _search_call_response() if len(seen_bodies) == 1 else _final_response()+ return httpx.Response(200, json=payload, headers={"content-type": "application/json"})++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ resp = _post(+ client,+ {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ "tools": _tools(13),+ "stream": False,+ },+ )++ assert resp.status_code == 200, resp.text+ # reload happened: two upstream calls, final answer returned to the client+ assert len(seen_bodies) == 2+ assert resp.json()["choices"][0]["message"]["content"] == "all done"+ assert hook.on_request_calls == 1+ assert hook.on_response_calls >= 1+ # shrink happened on the FIRST outbound body: 13 tools -> 2 (kept + search stub)+ first_tools = seen_bodies[0].get("tools")+ assert first_tools is not None and len(first_tools) == 2+ # the saving is surfaced as a transform+ transforms = resp.headers.get("x-headroom-transforms", "")+ assert "turn_hook" in transforms, transforms+++def test_saving_is_recorded_per_turn_and_aggregated_in_stats():+ """The deferred-tool-schema saving is recorded on EVERY turn (each request+ logs its own tag), and the dashboard's /stats sums them across turns."""+ register_turn_hook(_FakeRouterHook())++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ # no search_tools call -> no reload; just shrink + record per turn+ return httpx.Response(+ 200, json=_final_response(), headers={"content-type": "application/json"}+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ for _ in range(3): # three turns, same big tool belt each time+ r = _post(+ client,+ {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ "tools": _tools(13),+ "stream": False,+ },+ )+ assert r.status_code == 200, r.text++ # Every turn logged its own tool-schema saving.+ logs = client.app.state.proxy.logger.get_recent(10)+ saved_per_turn = [+ int((lg.get("tags") or {}).get("turn_hook_tools_saved_tokens", 0) or 0) for lg in logs+ ]+ assert sum(1 for s in saved_per_turn if s > 0) == 3, saved_per_turn++ # /stats aggregates the per-turn savings into the tool_search layer.+ stats = client.get("/stats").json()+ ts = stats["savings"]["by_layer"]["tool_search"]+ assert ts["requests"] == 3, ts+ assert ts["tokens"] == sum(saved_per_turn) > 0, (ts, saved_per_turn)+++def test_in_place_shrink_hook_is_counted():+ """The contract allows on_request to mutate ctx.tools IN PLACE (not just+ replace it). The saving must still be recorded even though the tools object+ identity is unchanged — regression for identity-gated savings accounting."""++ class InPlaceShrink:+ name = "inplace"++ def on_request(self, ctx):+ if isinstance(ctx.tools, list) and len(ctx.tools) > 2:+ # mutate the SAME list object (no reassignment)+ ctx.tools[:] = [+ ctx.tools[0],+ {"type": "function", "function": {"name": _SEARCH_TOOL}},+ ]++ register_turn_hook(InPlaceShrink())++ seen: list[dict] = []++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ import copy++ seen.append(copy.deepcopy(body))+ return httpx.Response(+ 200, json=_final_response(), headers={"content-type": "application/json"}+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ resp = _post(+ client,+ {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ "tools": _tools(13),+ "stream": False,+ },+ )+ assert resp.status_code == 200, resp.text+ # outbound request was shrunk in place (13 -> 2), same list object+ assert len(seen[0]["tools"]) == 2+ # ...and the saving is recorded despite the in-place mutation+ assert "turn_hook" in resp.headers.get("x-headroom-transforms", "")+ ts = client.get("/stats").json()["savings"]["by_layer"]["tool_search"]+ assert ts["tokens"] > 0 and ts["requests"] >= 1, ts+++def test_direct_path_noop_when_no_hook_registered():+ # No hook registered -> byte-identical passthrough, single upstream call.+ calls = {"n": 0}++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ calls["n"] += 1+ return httpx.Response(+ 200, json=_final_response(), headers={"content-type": "application/json"}+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ resp = _post(+ client,+ {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ "tools": _tools(13),+ "stream": False,+ },+ )++ assert resp.status_code == 200, resp.text+ assert calls["n"] == 1 # no reload+ assert resp.json()["choices"][0]["message"]["content"] == "all done"+ assert "turn_hook" not in resp.headers.get("x-headroom-transforms", "")