Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): ignore defer_loading when measuring a turn hook's tool delta (#4056)
Tejas Chopraf8212a39dcc5
diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.pyindex 8bbf41013..35248001c 100644--- a/headroom/proxy/gateway_turn.py+++ b/headroom/proxy/gateway_turn.py@@ -51,6 +51,7 @@ from typing import Anyfrom headroom.proxy.gateway_responses import VIEW_MARKER as GATEWAY_RESPONSES_VIEW_MARKERfrom headroom.proxy.outcome import RequestOutcome+from headroom.proxy.tool_schema_savings_policy import without_deferral_flagsfrom headroom.proxy.turn_hooks import (TurnContext,merge_provider_headers,@@ -893,7 +894,7 @@ class RequestTransformer:)if hooks:msg_before = _safe_count(count_messages, messages)- tools_before = _safe_count(count_tools, tools)+ tools_before = _safe_count(count_tools, without_deferral_flags(tools))deferred_before = self.tags.get("tool_search_deferred_tools")run_request_hooks(ctx, stream_safe_only=not self.caps.redrive_allowed)messages = ctx.messages@@ -911,7 +912,7 @@ class RequestTransformer:if msg_before is not None and msg_after is not None and msg_after < msg_before:transforms.append("turn_hook")messages_rewritten = True- tools_after = _safe_count(count_tools, tools)+ tools_after = _safe_count(count_tools, without_deferral_flags(tools))if tools_before is not None and tools_after is not None:saved = max(0, tools_before - tools_after)if saved > 0:diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex bcc738c34..8ec9c7b8d 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Anyfrom urllib.parse import urlsplitfrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log+from headroom.proxy.tool_schema_savings_policy import without_deferral_flagsif TYPE_CHECKING:from fastapi import Request@@ -3364,7 +3365,11 @@ class AnthropicHandlerMixin:except Exception:_pre_hook_tokens = None_th_tools_before = body.get("tools")- _th_tok_before = _count_tool_tokens(_th_tools_before) if _th_tools_before else 0+ _th_tok_before = (+ _count_tool_tokens(without_deferral_flags(_th_tools_before))+ if _th_tools_before+ else 0+ )run_request_hooks(_req_ctx, stream_safe_only=bool(stream))if _req_ctx.messages is not optimized_messages:optimized_messages = _req_ctx.messages@@ -3376,7 +3381,11 @@ class AnthropicHandlerMixin:# so measure the FINAL tools object. Deferral-shaped (removes schemas# count_messages never saw), hence a tag rather than a fold — mirrors# the OpenAI chat path so a turn-hook extension is credited on both.- _th_tok_after = _count_tool_tokens(_req_ctx.tools) if _req_ctx.tools else 0+ _th_tok_after = (+ _count_tool_tokens(without_deferral_flags(_req_ctx.tools))+ if _req_ctx.tools+ else 0+ )_th_saved = max(0, _th_tok_before - _th_tok_after)if _th_saved > 0:tags["turn_hook_tools_saved_tokens"] = (diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex beed80845..51f63ef89 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -39,6 +39,7 @@ from headroom.proxy.modes import is_cache_modefrom headroom.proxy.rate_limit_identity import rate_limit_identityfrom headroom.proxy.semantic_cache_key_policy import compute_request_cache_partitionfrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log+from headroom.proxy.tool_schema_savings_policy import without_deferral_flagsfrom headroom.proxy.upstream_guard import is_safe_upstream_urlfrom headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERSfrom headroom.proxy.ws_session_registry import (@@ -4833,7 +4834,9 @@ class OpenAIHandlerMixin:if registered_turn_hooks():_th_tools_before = body.get("tools")_th_tok_before = (- tokenizer.count_text(json.dumps(_th_tools_before, default=str))+ tokenizer.count_text(+ json.dumps(without_deferral_flags(_th_tools_before), default=str)+ )if _th_tools_beforeelse 0)@@ -4882,7 +4885,9 @@ class OpenAIHandlerMixin:except Exception:logger.debug("turn-hook token re-count skipped", exc_info=True)_th_tok_after = (- tokenizer.count_text(json.dumps(_th_ctx.tools, default=str)) if _th_ctx.tools else 0+ tokenizer.count_text(json.dumps(without_deferral_flags(_th_ctx.tools), default=str))+ if _th_ctx.tools+ else 0)_th_saved = max(0, _th_tok_before - _th_tok_after)if _th_saved > 0:diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.pyindex 5827d0c95..7b2bd8f81 100644--- a/headroom/proxy/tool_schema_savings_policy.py+++ b/headroom/proxy/tool_schema_savings_policy.py@@ -25,6 +25,113 @@ name to :data:`TOOL_SCHEMA_SAVINGS_TAGS` and every surface picks it up.from __future__ import annotations+import copy+from typing import Any+++def without_deferral_flags(tools: object) -> object:+ """The tool array as it is MEASURED for a hook's tool delta.++ Deferral is reported through ``tool_search_deferred_tokens`` (it removes+ schemas from the billed context while leaving them in the array, flagged+ ``defer_loading``). Measuring the raw array would see only the flag+ appear or disappear: un-deferring a tool would shorten the JSON by the+ flag text and be booked as a small saving although the bill goes UP, and+ deferring one would add a phantom cost. So the flag is ignored here, and a+ hook's measured tool delta covers only real removals and rewrites. Never+ mutates its input.+ """+ if not isinstance(tools, list):+ return tools+ return [+ {k: v for k, v in tool.items() if k != "defer_loading"}+ if isinstance(tool, dict) and "defer_loading" in tool+ else tool+ for tool in tools+ ]+++DeferredBooking = tuple[int, dict[str, Any], "dict[str, Any] | None"]+++def deferred_booking(tags: object, tools: object) -> DeferredBooking | None:+ """Snapshot the deferral currently booked: the tag, the tools it covers and+ its own ledger entry.++ ``None`` when nothing is booked. The schemas are copied as they were when+ booked, so a later in-place edit cannot change what the credit stands for.+ The ledger entry is the latest ``tool_search`` one marked ``estimated``:+ both the core deferral and the native deferral hook book that way, and a+ hook's own measured saving never is, so it is never the one debited.+ """+ if not isinstance(tags, dict) or "tool_search_deferred_tokens" not in tags:+ return None+ try:+ booked = int(tags.get("tool_search_deferred_tokens") or 0)+ except (TypeError, ValueError):+ return None+ snapshot = {+ str(t["name"]): copy.deepcopy(t)+ for t in (tools if isinstance(tools, list) else [])+ if isinstance(t, dict) and t.get("defer_loading") and t.get("name")+ }+ from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG++ ledger = tags.get(SAVINGS_ATTRIBUTION_TAG)+ entry = next(+ (+ item+ for item in reversed(ledger if isinstance(ledger, list) else [])+ if isinstance(item, dict)+ and item.get("source") == "tool_search"+ and item.get("estimated")+ ),+ None,+ )+ return booked, snapshot, entry+++def reconcile_deferred_tokens(+ tags: object, booking: DeferredBooking | None, tools: object, count_tools: object+) -> None:+ """Cap the deferral credit at the booked tools still deferred after hooks.++ A hook may un-defer tools the deferral booked (tool search's hot tools);+ those are billed, so their share of the credit goes. Each tool still+ deferred is credited at the SMALLER of its booked and its final schema:+ growing another schema cannot keep an un-deferred tool's credit alive, and+ a schema a hook shortened (and booked as its own saving) is not credited+ twice. Only ever lowers the tag, and lowers the deferral's own ledger entry+ by the same amount so ``/stats`` by-source agrees with the headline.+ Never raises.+ """+ if booking is None or not isinstance(tags, dict) or not callable(count_tools):+ return+ booked, snapshot, entry = booking+ try:+ final = {+ str(t["name"]): t+ for t in (tools if isinstance(tools, list) else [])+ if isinstance(t, dict) and t.get("defer_loading") and t.get("name")+ }+ kept = []+ for name, schema in snapshot.items():+ if name not in final:+ continue+ now = final[name]+ kept.append(now if int(count_tools([now])) < int(count_tools([schema])) else schema)+ measured = int(count_tools(kept)) if kept else 0+ released = booked - measured+ if released <= 0:+ return+ tags["tool_search_deferred_tokens"] = measured+ tags["tool_search_deferred_tools"] = len(kept)+ if isinstance(entry, dict):+ entry["tokens"] = max(0, int(entry.get("tokens") or 0) - released)+ except Exception: # accounting must never break a request+ return++TOOL_SCHEMA_SAVINGS_TAGS: tuple[str, ...] = ("tool_search_deferred_tokens","turn_hook_tools_saved_tokens",diff --git a/headroom/proxy/turn_hooks.py b/headroom/proxy/turn_hooks.pyindex 1b10aeaa8..6e4987075 100644--- a/headroom/proxy/turn_hooks.py+++ b/headroom/proxy/turn_hooks.py@@ -26,6 +26,12 @@ from collections.abc import Awaitable, Callablefrom dataclasses import dataclass, fieldfrom typing import Any, Protocol, runtime_checkable+from headroom.proxy.tool_schema_savings_policy import (+ deferred_booking,+ reconcile_deferred_tokens,+ without_deferral_flags,+)+log = logging.getLogger(__name__)# Re-drive the model with a message list; returns the provider's response JSON.@@ -198,18 +204,22 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> Noworking on streamed OpenAI-compatible traffic. Default off ⇒ conservative:a hook is treated as buffered-only unless it declares itself stream-safe."""+ # Deferral booked before or by a hook; reconciled once every hook has run,+ # since a later hook may un-defer booked tools (tool search's hot tools).+ booking = deferred_booking(ctx.tags, ctx.tools)for hook in registered_turn_hooks():if stream_safe_only and not getattr(hook, "stream_safe", False):continuefn = getattr(hook, "on_request", None)if fn is None:continue+ booked_tag = ctx.tags.get("tool_search_deferred_tokens")before_messages = before_tools = Nonetry:if ctx.count_messages is not None:before_messages = ctx.count_messages(ctx.messages)if ctx.count_tools is not None:- before_tools = ctx.count_tools(ctx.tools)+ before_tools = ctx.count_tools(without_deferral_flags(ctx.tools))fn(ctx)message_saved = (max(0, before_messages - ctx.count_messages(ctx.messages))@@ -217,7 +227,7 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> Noelse 0)tool_saved = (- max(0, before_tools - ctx.count_tools(ctx.tools))+ max(0, before_tools - ctx.count_tools(without_deferral_flags(ctx.tools)))if before_tools is not None and ctx.count_tools is not Noneelse 0)@@ -232,6 +242,10 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> No)except Exception: # a hook must never break the proxylog.exception("turn hook %r on_request failed", getattr(hook, "name", hook))+ if ctx.tags.get("tool_search_deferred_tokens") != booked_tag:+ # This hook booked (or re-booked) the deferral: it is the new baseline.+ booking = deferred_booking(ctx.tags, ctx.tools)+ reconcile_deferred_tokens(ctx.tags, booking, ctx.tools, ctx.count_tools)async def run_response_hooks(diff --git a/tests/test_hook_tool_deferral_accounting.py b/tests/test_hook_tool_deferral_accounting.pynew file mode 100644index 000000000..9077218fb--- /dev/null+++ b/tests/test_hook_tool_deferral_accounting.py@@ -0,0 +1,300 @@+"""A turn hook's measured tool delta ignores the ``defer_loading`` flag.++Deferral is booked through ``tool_search_deferred_tokens``. Measuring the raw+tool array would see only the flag text appear or disappear, so un-deferring a+tool (tool search's hot-tools hook) was booked as a realized saving although+it makes the request larger.+"""++from __future__ import annotations++import json++import pytest++from headroom.proxy.savings_attribution import from_tags, record_savings+from headroom.proxy.tool_schema_savings_policy import without_deferral_flags+from headroom.proxy.turn_hooks import (+ TurnContext,+ clear_turn_hooks,+ register_turn_hook,+ run_request_hooks,+)++[email protected](autouse=True)+def _clean_registry():+ clear_turn_hooks()+ yield+ clear_turn_hooks()+++def _tool(name: str, size: int = 400) -> dict:+ return {"name": name, "description": "d" * size, "input_schema": {"type": "object"}}+++def _count(value: object) -> int:+ # Stand-in tokenizer: one token per 4 characters of the serialised array.+ return len(json.dumps(value, default=str)) // 4 if value else 0+++class _UndeferHook:+ """Shaped like headroom-tool-search's HotToolsHook: drops the flag in place."""++ name = savings_source = "tool_search"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ for tool in ctx.tools:+ tool.pop("defer_loading", None)+++class _DeferHook:+ """Shaped like NativeDeferralHook: flags tools, books through the tag."""++ name = savings_source = "tool_search"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ ctx.tools = [{**t, "defer_loading": True} for t in ctx.tools]+++class _RemoveHook:+ """Shaped like a router: really removes a tool from the array."""++ name = savings_source = "tool_router"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ ctx.tools = ctx.tools[:1]+++def _run(hook: object, tools: list) -> list[dict]:+ register_turn_hook(hook)+ ctx = TurnContext(+ provider="anthropic",+ model="claude-sonnet-4-6",+ messages=[{"role": "user", "content": "hi"}],+ tools=tools,+ count_messages=_count,+ count_tools=_count,+ )+ run_request_hooks(ctx)+ return from_tags(ctx.tags)+++def test_undeferring_a_tool_is_not_booked_as_a_saving() -> None:+ tools = [{**_tool("a"), "defer_loading": True}, {**_tool("b"), "defer_loading": True}]+ assert _run(_UndeferHook(), tools) == []+++def test_deferring_is_left_to_the_deferral_tag() -> None:+ # Only the flag changes, so the measured delta is zero and the hook's own+ # tool_search_deferred_tokens tag stays the single place deferral is booked.+ assert _run(_DeferHook(), [_tool("a"), _tool("b")]) == []+++def test_a_real_removal_is_still_measured() -> None:+ (entry,) = _run(_RemoveHook(), [_tool("a"), _tool("b")])+ assert entry["source"] == "tool_router"+ assert entry["tokens"] == _count([_tool("a"), _tool("b")]) - _count([_tool("a")])+++def test_without_deferral_flags_never_mutates_and_passes_non_lists() -> None:+ tools = [{**_tool("a"), "defer_loading": True}, "not-a-dict"]+ out = without_deferral_flags(tools)+ assert "defer_loading" in tools[0]+ assert out[0] == _tool("a")+ assert out[1] == "not-a-dict"+ assert without_deferral_flags(None) is None+++# ── deferral credit is capped at what is still deferred after hooks ──────────+++class _BookDeferral:+ """Shaped like the native deferral hook: flags tools and books them."""++ name = savings_source = "tool_search"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ ctx.tools = [{**t, "defer_loading": True} for t in ctx.tools]+ ctx.tags["tool_search_deferred_tokens"] = _count(ctx.tools)+ ctx.tags["tool_search_deferred_tools"] = len(ctx.tools)+ record_savings(ctx.tags, "tool_search", tokens=_count(ctx.tools), estimated=True)+++class _UndeferFirstGrowSecond:+ name = savings_source = "other"+ stream_safe = True++ def __init__(self, grow: int = 0) -> None:+ self.grow = grow++ def on_request(self, ctx: TurnContext) -> None:+ ctx.tools[0].pop("defer_loading", None)+ ctx.tools[1]["description"] += "g" * self.grow+++def _booked(*hooks: object) -> tuple[dict, list]:+ for hook in hooks:+ register_turn_hook(hook)+ tags: dict = {}+ ctx = TurnContext(+ provider="anthropic",+ model="claude-sonnet-4-6",+ messages=[{"role": "user", "content": "hi"}],+ tools=[_tool("a"), _tool("b")],+ count_messages=_count,+ count_tools=_count,+ tags=tags,+ )+ run_request_hooks(ctx)+ return tags, [e for e in from_tags(tags) if e["source"] == "tool_search"]+++def test_undeferring_a_booked_tool_lowers_the_headline_and_the_ledger() -> None:+ tags, (entry,) = _booked(_BookDeferral(), _UndeferFirstGrowSecond())+ expected = _count([{**_tool("b"), "defer_loading": True}])+ assert tags["tool_search_deferred_tokens"] == expected+ assert tags["tool_search_deferred_tools"] == 1+ assert entry["tokens"] == expected # /stats by-source agrees with the headline+++def test_growing_another_deferred_schema_cannot_keep_the_credit() -> None:+ # b grows by far more than a's share: the final deferred set outweighs the+ # booking, yet a is billed now, so its share must still go.+ tags, (entry,) = _booked(_BookDeferral(), _UndeferFirstGrowSecond(grow=4_000))+ expected = _count([{**_tool("b"), "defer_loading": True}])+ assert tags["tool_search_deferred_tokens"] == expected+ assert entry["tokens"] == expected+++def test_reconcile_never_raises_the_credit_or_touches_unbooked_turns() -> None:+ tags, (entry,) = _booked(_BookDeferral())+ both = _count([{**_tool("a"), "defer_loading": True}, {**_tool("b"), "defer_loading": True}])+ assert tags["tool_search_deferred_tokens"] == both == entry["tokens"]+ clear_turn_hooks()+ untouched, _ = _booked(_UndeferFirstGrowSecond())+ assert "tool_search_deferred_tokens" not in untouched+++def test_anthropic_headline_drops_the_credit_of_a_tool_a_hook_undefers(monkeypatch) -> None:+ """End to end: the built-in deferral books 14 MCP tools, then a hook+ (shaped like tool search's hot tools) un-defers one. The headline credit+ must cover only the 13 still deferred."""+ import httpx+ import respx++ pytest.importorskip("fastapi")+ from fastapi.testclient import TestClient++ from headroom.proxy.loopback_guard import require_loopback+ from headroom.proxy.server import ProxyConfig, create_app+ from headroom.tokenizers import get_tokenizer++ monkeypatch.setenv("HEADROOM_TOOL_SEARCH", "1")++ class _UndeferOne:+ name = savings_source = "tool_search"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ for tool in ctx.tools:+ if isinstance(tool, dict) and tool.get("name") == "mcp__srv__tool_0":+ tool.pop("defer_loading", None)++ register_turn_hook(_UndeferOne())+ app = create_app(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ )+ )+ app.dependency_overrides[require_loopback] = lambda: None+ outcomes: list = []++ async def _spy(_self, outcome, *a, **kw): # noqa: ANN001, ANN002, ANN003, ANN202+ outcomes.append(outcome)++ monkeypatch.setattr(type(app.state.proxy), "_record_request_outcome", _spy, raising=True)+ tools = [+ {+ "name": f"mcp__srv__tool_{i}",+ "description": "Look things up. " * 30,+ "input_schema": {"type": "object"},+ }+ for i in range(14)+ ]+ with respx.mock:+ respx.post("https://api.anthropic.com/v1/messages").mock(+ return_value=httpx.Response(+ 200,+ json={+ "id": "a",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-5",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 100, "output_tokens": 1},+ },+ )+ )+ with TestClient(app) as client:+ result = client.post(+ "/v1/messages",+ json={+ "model": "claude-sonnet-4-5",+ "max_tokens": 16,+ "messages": [{"role": "user", "content": "hi"}],+ "tools": tools,+ },+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},+ )+ assert result.status_code == 200+ tags = outcomes[-1].tags+ assert tags.get("tool_search_mode") == "headroom"+ still = [{**t, "defer_loading": True} for t in tools[1:]]+ tok = get_tokenizer("claude-sonnet-4-5")+ assert tags["tool_search_deferred_tools"] == 13+ assert tags["tool_search_deferred_tokens"] == tok.count_text(json.dumps(still, default=str))+ (entry,) = [e for e in from_tags(tags) if e["source"] == "tool_search"]+ assert entry["tokens"] == tags["tool_search_deferred_tokens"]+++class _ShrinkSecondUndeferFirst:+ """A tool_search-sourced hook: shortens b (still deferred), un-defers a."""++ name = savings_source = "tool_search"+ stream_safe = True++ def on_request(self, ctx: TurnContext) -> None:+ ctx.tools[0].pop("defer_loading", None)+ ctx.tools[1]["description"] = "short"+++def test_release_debits_the_deferral_entry_not_a_hooks_own_saving() -> None:+ tags, entries = _booked(_BookDeferral(), _ShrinkSecondUndeferFirst())+ deferral = [e for e in entries if e.get("estimated")]+ measured = [e for e in entries if not e.get("estimated")]+ (deferral_entry,) = deferral+ (hook_entry,) = measured+ shrunk = {**_tool("b"), "description": "short"}+ # The hook keeps its own measured saving (b's shrink, flags ignored)...+ assert hook_entry["tokens"] == _count([_tool("a"), _tool("b")]) - _count([_tool("a"), shrunk])+ # ...and only the deferral entry is debited, down to what is still deferred.+ assert deferral_entry["tokens"] == tags["tool_search_deferred_tokens"]+++def test_a_shortened_deferred_schema_is_credited_once() -> None:+ # b's shrink is the hook's saving; the deferral credits b at its final+ # (smaller) size, so those tokens are not counted twice.+ tags, _ = _booked(_BookDeferral(), _ShrinkSecondUndeferFirst())+ shrunk = {**_tool("b"), "description": "short", "defer_loading": True}+ assert tags["tool_search_deferred_tokens"] == _count([shrunk])+ assert tags["tool_search_deferred_tools"] == 1