Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Tejas/turn hooks extension (#1903)

Tejas Choprac9217856d3c2
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.html
index f5aa65d3c..54d04c240 100644
--- a/headroom/dashboard/templates/dashboard.html
+++ b/headroom/dashboard/templates/dashboard.html
@@ -269,6 +269,20 @@
</template>
</div>
+ <!-- Tool-Schema Deferral (tool search) — only rendered when there's a saving to show -->
+ <template x-if="(stats.savings?.by_layer?.tool_search?.tokens || 0) > 0">
+ <div class="bg-surface rounded-lg p-4 border border-border">
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tool-Schema Deferral</div>
+ <div class="flex items-baseline gap-2">
+ <span class="text-3xl font-light tabular-nums text-emerald-400" x-text="formatNumber(stats.savings?.by_layer?.tool_search?.tokens || 0)"></span>
+ <span class="text-sm text-gray-400">tokens</span>
+ </div>
+ <div class="mt-1 text-xs text-gray-600 leading-relaxed"
+ x-text="formatNumber(stats.savings?.by_layer?.tool_search?.requests || 0) + ' calls · tool schemas deferred (recent window)'">
+ </div>
+ </div>
+ </template>
+
<!-- Headroom Overhead -->
<div class="bg-surface rounded-lg p-4 border border-border">
<div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Overhead</div>
@@ -2002,6 +2016,7 @@
<span class="text-xs text-gray-600 shrink-0">·</span>
<span class="text-xs text-emerald-400 shrink-0">${tokensSaved} tok</span>
<span class="text-xs text-gray-600 shrink-0">(${savingsPct}%)</span>
+ ${(t.tool_schema_saved_tokens || 0) > 0 ? `<span class="text-xs text-cyan-400 shrink-0" title="Tool-definition tokens deferred out of context">+${t.tool_schema_saved_tokens} tool-schema</span>` : ''}
</div>
<span class="text-xs text-gray-600 shrink-0">${time}</span>
</div>
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 3e5ff61f4..86a12ce43 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -2758,6 +2758,52 @@ class OpenAIHandlerMixin:
optimized_tokens = tokenizer.count_messages(body["messages"])
tokens_saved = original_tokens - optimized_tokens
+ # Turn hooks (opt-in extensions): a registered hook may rewrite the
+ # outbound tools/messages before we send. Buffered requests only — a
+ # streamed turn can't be re-driven to resolve whatever the model asks to
+ # load. Gated on the registry so it is a no-op when none are registered;
+ # the net tool-schema token delta is recorded so it shows up as a saving.
+ from headroom.proxy.turn_hooks import (
+ TurnContext,
+ registered_turn_hooks,
+ run_request_hooks,
+ )
+
+ if registered_turn_hooks() and not stream:
+ _th_tools_before = body.get("tools")
+ _th_tok_before = (
+ tokenizer.count_text(json.dumps(_th_tools_before, default=str))
+ if _th_tools_before
+ else 0
+ )
+ _th_ctx = TurnContext(
+ provider="openai",
+ model=str(model),
+ messages=body["messages"],
+ tools=_th_tools_before,
+ config=self.config,
+ )
+ run_request_hooks(_th_ctx)
+ # A hook may either replace ctx.messages/ctx.tools or mutate them in
+ # place (the contract allows both). Use object identity only to decide
+ # whether body needs reassignment; measure the saving from the FINAL
+ # tools object regardless, so an in-place shrink is still counted.
+ if _th_ctx.messages is not body["messages"]:
+ optimized_messages = _th_ctx.messages
+ body["messages"] = optimized_messages
+ if _th_ctx.tools is not _th_tools_before:
+ tools = _th_ctx.tools
+ body["tools"] = tools
+ _th_tok_after = (
+ tokenizer.count_text(json.dumps(_th_ctx.tools, default=str)) if _th_ctx.tools else 0
+ )
+ _th_saved = max(0, _th_tok_before - _th_tok_after)
+ if _th_saved > 0:
+ tags["turn_hook_tools_saved_tokens"] = (
+ int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved
+ )
+ transforms_applied.append(f"turn_hook:tools:{_th_saved}tok")
+
# Compatibility shim: GPT-5 / o-series chat models REJECT the legacy
# `max_tokens` ("Unsupported parameter … Use 'max_completion_tokens'
# instead"); gpt-4o/4.1 accept `max_completion_tokens` too. openai-
@@ -3071,6 +3117,54 @@ class OpenAIHandlerMixin:
else:
headers = await apply_copilot_api_auth(headers, url=url)
response = await self._retry_request("POST", url, headers, body)
+
+ # Turn hooks: a registered extension may re-drive this turn
+ # (e.g. resolve a tool the model asked to load) before we treat
+ # the response as final. Buffered path only; no-op when no hook
+ # is registered.
+ from headroom.proxy.turn_hooks import (
+ TurnContext as _TurnContext,
+ )
+ from headroom.proxy.turn_hooks import (
+ registered_turn_hooks as _registered_turn_hooks,
+ )
+ from headroom.proxy.turn_hooks import (
+ run_response_hooks,
+ )
+
+ if _registered_turn_hooks() and response.status_code == 200:
+ try:
+ _hook_resp_json = response.json()
+ except (ValueError, json.JSONDecodeError):
+ _hook_resp_json = None
+ if isinstance(_hook_resp_json, dict):
+ _hook_ctx = _TurnContext(
+ provider="openai",
+ model=str(model),
+ messages=body["messages"],
+ tools=body.get("tools"),
+ config=self.config,
+ )
+
+ async def _hook_call_model(_msgs):
+ body["messages"] = _msgs
+ _r = await self._retry_request("POST", url, headers, body)
+ return _r.json()
+
+ _hook_final = await run_response_hooks(
+ _hook_ctx, _hook_resp_json, _hook_call_model
+ )
+ if _hook_final is not _hook_resp_json:
+ response = httpx.Response(
+ status_code=200,
+ headers={
+ k: v
+ for k, v in response.headers.items()
+ if k.lower() not in ("content-encoding", "content-length")
+ },
+ content=json.dumps(_hook_final).encode(),
+ )
+
self.pipeline_extensions.emit(
PipelineStage.POST_SEND,
operation="proxy.request",
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 1759627b0..3d69e5ff6 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -2012,6 +2012,24 @@ def _is_known_websocket_callback_failure(context: dict[str, Any]) -> bool:
)
+def _tool_schema_saved_from_tags(tags: object) -> int:
+ """Tool-definition tokens Headroom kept out of context for one request by
+ deferring heavy tool schemas: the native tool-search injection
+ (``tool_search_deferred_tokens``) plus any registered turn-hook tools
+ rewrite (``turn_hook_tools_saved_tokens``). Both tags are set only on the
+ path where Headroom performed the deferral, so a client that already had
+ tool search enabled (e.g. Claude Code / Codex) contributes zero here."""
+ if not isinstance(tags, dict):
+ return 0
+ total = 0
+ for key in ("tool_search_deferred_tokens", "turn_hook_tools_saved_tokens"):
+ try:
+ total += int(tags.get(key, 0) or 0)
+ except (TypeError, ValueError):
+ continue
+ return total
+
+
def create_app(config: ProxyConfig | None = None) -> FastAPI:
"""Create FastAPI application."""
if not FASTAPI_AVAILABLE:
@@ -2945,6 +2963,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"total_latency_ms": log.get("total_latency_ms"),
"transforms_applied": log.get("transforms_applied", []),
"waste_signals": log.get("waste_signals"),
+ "tool_schema_saved_tokens": _tool_schema_saved_from_tags(log.get("tags")),
}
for log in recent_request_logs
if log.get("input_tokens_original") is not None
@@ -3129,6 +3148,19 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
display_session = persistent_savings.get("display_session", {})
recent_request_logs = proxy.logger.get_recent(10_000) if proxy.logger else []
recent_request_payload = _build_recent_request_payload()
+
+ # Tool-schema deferral savings: tool-definition tokens kept out of the
+ # model's context by deferring heavy schemas until they're needed
+ # (native tool-search injection + any registered turn-hook tools
+ # rewrite). Attributed to Headroom only — see _tool_schema_saved_from_tags.
+ # Aggregated over the recent request-log window.
+ tool_schema_tokens = 0
+ tool_schema_requests = 0
+ for _ts_log in recent_request_logs:
+ _ts_saved = _tool_schema_saved_from_tags(_ts_log.get("tags"))
+ if _ts_saved > 0:
+ tool_schema_tokens += _ts_saved
+ tool_schema_requests += 1
agent_usage = _build_agent_usage_summary(
recent_request_logs,
requests_by_provider=dict(m.requests_by_provider),
@@ -3229,6 +3261,20 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"(A/B holdout), always with a confidence band."
),
},
+ "tool_search": {
+ "tokens": tool_schema_tokens,
+ "tokens_saved": tool_schema_tokens,
+ "requests": tool_schema_requests,
+ "window": len(recent_request_logs),
+ "description": (
+ "Tool-definition tokens kept out of the model's context "
+ "by deferring heavy tool schemas until they're searched "
+ "for. Counted only when Headroom performed the deferral — "
+ "not when the client (e.g. Claude Code / Codex) already "
+ "had tool search enabled. Aggregated over the recent "
+ "request window."
+ ),
+ },
},
},
"requests": {
diff --git a/tests/test_openai_chat_turn_hooks.py b/tests/test_openai_chat_turn_hooks.py
new file mode 100644
index 000000000..15f6b27d0
--- /dev/null
+++ b/tests/test_openai_chat_turn_hooks.py
@@ -0,0 +1,295 @@
+"""End-to-end turn-hook wiring on the OpenAI chat-completions direct path.
+
+Proves the two seams added to ``handle_openai_chat`` for the direct
+(no-backend) buffered path:
+
+* ``on_request`` fires before the upstream send — a hook can shrink the
+ outbound ``tools``, and the net tool-schema token delta is recorded as a
+ saving (surfaced via the ``x-headroom-transforms`` header / tags).
+* ``on_response`` fires after the send with a working ``call_model`` — a hook
+ can detect a tool the model asked to load, re-drive the model, and have the
+ proxy return the *final* response transparently.
+
+Uses a fake hook (mimicking the tool-router extension's shrink + reload) and a
+mocked ``_retry_request`` so no network / real provider is needed. Also pins the
+no-op property: with no hook registered the path is unchanged.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook # noqa: E402
+
+_SEARCH_TOOL = "search_tools"
+
+
[email protected](autouse=True)
+def _clean_hooks():
+ clear_turn_hooks()
+ yield
+ clear_turn_hooks()
+
+
+def _big_tool(name: str) -> dict:
+ return {
+ "type": "function",
+ "function": {
+ "name": name,
+ "description": f"{name} does a thing " + ("x " * 40),
+ "parameters": {
+ "type": "object",
+ "properties": {"arg": {"type": "string", "description": "y " * 60}},
+ },
+ },
+ }
+
+
+def _tools(n: int = 13) -> list[dict]:
+ return [_big_tool(f"tool_{i}") for i in range(n)]
+
+
+def _search_call_response() -> dict:
+ return {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": _SEARCH_TOOL,
+ "arguments": '{"query":"do a thing"}',
+ },
+ }
+ ],
+ },
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
+ }
+
+
+def _final_response() -> dict:
+ return {
+ "id": "chatcmpl-2",
+ "object": "chat.completion",
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "all done"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 120, "completion_tokens": 5, "total_tokens": 125},
+ }
+
+
+class _FakeRouterHook:
+ """Mimics the tool-router extension: shrink on request, reload on response."""
+
+ name = "fake_router"
+
+ def __init__(self):
+ self.on_request_calls = 0
+ self.on_response_calls = 0
+
+ def on_request(self, ctx):
+ self.on_request_calls += 1
+ # Shrink: drop all but the first tool + inject a search_tools stub.
+ if isinstance(ctx.tools, list) and len(ctx.tools) > 2:
+ ctx.tools = [ctx.tools[0], {"type": "function", "function": {"name": _SEARCH_TOOL}}]
+
+ async def on_response(self, ctx, response, call_model):
+ self.on_response_calls += 1
+ tcs = (response.get("choices") or [{}])[0].get("message", {}).get("tool_calls") or []
+ if any(tc.get("function", {}).get("name") == _SEARCH_TOOL for tc in tcs):
+ return await call_model(ctx.messages + [{"role": "user", "content": "resolved"}])
+ return None
+
+
+def _config() -> ProxyConfig:
+ # No backend -> the "Direct OpenAI API (no backend configured)" path.
+ return ProxyConfig(optimize=False, cache_enabled=False, rate_limit_enabled=False)
+
+
+def _post(client: TestClient, body: dict):
+ return client.post(
+ "/v1/chat/completions",
+ json=body,
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+
+def test_direct_path_shrinks_then_reloads_and_returns_final():
+ hook = _FakeRouterHook()
+ register_turn_hook(hook)
+
+ seen_bodies: list[dict] = []
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs):
+ # capture the exact outbound body per upstream call
+ import copy
+
+ seen_bodies.append(copy.deepcopy(body))
+ payload = _search_call_response() if len(seen_bodies) == 1 else _final_response()
+ return httpx.Response(200, json=payload, headers={"content-type": "application/json"})
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ client.app.state.proxy._retry_request = fake_retry
+ resp = _post(
+ client,
+ {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": _tools(13),
+ "stream": False,
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ # reload happened: two upstream calls, final answer returned to the client
+ assert len(seen_bodies) == 2
+ assert resp.json()["choices"][0]["message"]["content"] == "all done"
+ assert hook.on_request_calls == 1
+ assert hook.on_response_calls >= 1
+ # shrink happened on the FIRST outbound body: 13 tools -> 2 (kept + search stub)
+ first_tools = seen_bodies[0].get("tools")
+ assert first_tools is not None and len(first_tools) == 2
+ # the saving is surfaced as a transform
+ transforms = resp.headers.get("x-headroom-transforms", "")
+ assert "turn_hook" in transforms, transforms
+
+
+def test_saving_is_recorded_per_turn_and_aggregated_in_stats():
+ """The deferred-tool-schema saving is recorded on EVERY turn (each request
+ logs its own tag), and the dashboard's /stats sums them across turns."""
+ register_turn_hook(_FakeRouterHook())
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs):
+ # no search_tools call -> no reload; just shrink + record per turn
+ return httpx.Response(
+ 200, json=_final_response(), headers={"content-type": "application/json"}
+ )
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ client.app.state.proxy._retry_request = fake_retry
+ for _ in range(3): # three turns, same big tool belt each time
+ r = _post(
+ client,
+ {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": _tools(13),
+ "stream": False,
+ },
+ )
+ assert r.status_code == 200, r.text
+
+ # Every turn logged its own tool-schema saving.
+ logs = client.app.state.proxy.logger.get_recent(10)
+ saved_per_turn = [
+ int((lg.get("tags") or {}).get("turn_hook_tools_saved_tokens", 0) or 0) for lg in logs
+ ]
+ assert sum(1 for s in saved_per_turn if s > 0) == 3, saved_per_turn
+
+ # /stats aggregates the per-turn savings into the tool_search layer.
+ stats = client.get("/stats").json()
+ ts = stats["savings"]["by_layer"]["tool_search"]
+ assert ts["requests"] == 3, ts
+ assert ts["tokens"] == sum(saved_per_turn) > 0, (ts, saved_per_turn)
+
+
+def test_in_place_shrink_hook_is_counted():
+ """The contract allows on_request to mutate ctx.tools IN PLACE (not just
+ replace it). The saving must still be recorded even though the tools object
+ identity is unchanged — regression for identity-gated savings accounting."""
+
+ class InPlaceShrink:
+ name = "inplace"
+
+ def on_request(self, ctx):
+ if isinstance(ctx.tools, list) and len(ctx.tools) > 2:
+ # mutate the SAME list object (no reassignment)
+ ctx.tools[:] = [
+ ctx.tools[0],
+ {"type": "function", "function": {"name": _SEARCH_TOOL}},
+ ]
+
+ register_turn_hook(InPlaceShrink())
+
+ seen: list[dict] = []
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs):
+ import copy
+
+ seen.append(copy.deepcopy(body))
+ return httpx.Response(
+ 200, json=_final_response(), headers={"content-type": "application/json"}
+ )
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ client.app.state.proxy._retry_request = fake_retry
+ resp = _post(
+ client,
+ {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": _tools(13),
+ "stream": False,
+ },
+ )
+ assert resp.status_code == 200, resp.text
+ # outbound request was shrunk in place (13 -> 2), same list object
+ assert len(seen[0]["tools"]) == 2
+ # ...and the saving is recorded despite the in-place mutation
+ assert "turn_hook" in resp.headers.get("x-headroom-transforms", "")
+ ts = client.get("/stats").json()["savings"]["by_layer"]["tool_search"]
+ assert ts["tokens"] > 0 and ts["requests"] >= 1, ts
+
+
+def test_direct_path_noop_when_no_hook_registered():
+ # No hook registered -> byte-identical passthrough, single upstream call.
+ calls = {"n": 0}
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs):
+ calls["n"] += 1
+ return httpx.Response(
+ 200, json=_final_response(), headers={"content-type": "application/json"}
+ )
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ client.app.state.proxy._retry_request = fake_retry
+ resp = _post(
+ client,
+ {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": _tools(13),
+ "stream": False,
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert calls["n"] == 1 # no reload
+ assert resp.json()["choices"][0]["message"]["content"] == "all done"
+ assert "turn_hook" not in resp.headers.get("x-headroom-transforms", "")