Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): net tools a turn hook adds against the saving it books (#4057)

Tejas Chopra85520c2941ed
diff --git a/headroom/proxy/compress_turn.py b/headroom/proxy/compress_turn.py
index 1db795712..c5f6a7826 100644
--- a/headroom/proxy/compress_turn.py
+++ b/headroom/proxy/compress_turn.py
@@ -89,6 +89,14 @@ class CompressTurn(Protocol):
def count_messages(self, messages: list[dict[str, Any]], fallback: int) -> int:
"""Recount with the turn's own tokenizer; ``fallback`` on failure."""
+ # Optional, not part of the required protocol: a turn may expose
+ # ``tool_growth_tokens``, the tool-definition tokens ``transform`` ADDED to
+ # the request (a hook's search tool, say). The handler adds it to
+ # ``tokens_after``, since those tokens are sent too; absent reads as 0.
+ # Likewise optional: ``message_growth_tokens`` (message tokens it added)
+ # and ``hook_tool_saved_tokens`` (tool tokens it removed and booked), so
+ # the handler can net the two before clamping the saving.
+
def finish(
self,
*,
diff --git a/headroom/proxy/gateway_extension.py b/headroom/proxy/gateway_extension.py
index 90c18ddf0..3d61620ba 100644
--- a/headroom/proxy/gateway_extension.py
+++ b/headroom/proxy/gateway_extension.py
@@ -141,6 +141,18 @@ class GatewayTurn:
return fallback
return self._transformer.count_messages(messages, fallback)
+ @property
+ def tool_growth_tokens(self) -> int:
+ return self._transformer.tool_growth_tokens if self._transformer is not None else 0
+
+ @property
+ def message_growth_tokens(self) -> int:
+ return self._transformer.message_growth_tokens if self._transformer is not None else 0
+
+ @property
+ def hook_tool_saved_tokens(self) -> int:
+ return self._transformer.hook_tool_saved_tokens if self._transformer is not None else 0
+
# -- event-loop side ---------------------------------------------------
def finish(
self,
diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.py
index bc7e31021..46a7b68a0 100644
--- a/headroom/proxy/gateway_turn.py
+++ b/headroom/proxy/gateway_turn.py
@@ -839,6 +839,14 @@ class RequestTransformer:
self.tags = tags
self.caps = caps
self.result: RequestTransformResult | None = None
+ # Tool-definition tokens the request hooks ADDED (e.g. a search tool).
+ # The compress handler adds it to ``tokens_after``: it is sent too.
+ self.tool_growth_tokens = 0
+ # Message tokens the request hooks ADDED (only counted by the handler
+ # when it did not recount the messages), and tool tokens they removed
+ # (booked on turn_hook_tools_saved_tokens), for signed netting.
+ self.message_growth_tokens = 0
+ self.hook_tool_saved_tokens = 0
self._tokenizer: Any = None
def _get_tokenizer(self) -> Any:
@@ -909,12 +917,16 @@ class RequestTransformer:
f"{int(self.tags.get('tool_search_deferred_tokens', 0) or 0)}tok"
)
msg_after = _safe_count(count_messages, messages)
+ if msg_before is not None and msg_after is not None:
+ self.message_growth_tokens = max(0, msg_after - msg_before)
if msg_before is not None and msg_after is not None and msg_after < msg_before:
transforms.append("turn_hook")
messages_rewritten = True
tools_after = _safe_count(count_tools, without_deferral_flags(tools))
if tools_before is not None and tools_after is not None:
+ self.tool_growth_tokens = max(0, tools_after - tools_before)
saved = max(0, tools_before - tools_after)
+ self.hook_tool_saved_tokens = saved
if saved > 0:
self.tags["turn_hook_tools_saved_tokens"] = (
int(self.tags.get("turn_hook_tools_saved_tokens", 0) or 0) + saved
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 3de6f8295..a5232b2ae 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -21,7 +21,10 @@ from urllib.parse import urlsplit
from headroom.proxy.session_affinity import session_affinity_headers
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
-from headroom.proxy.tool_schema_savings_policy import without_deferral_flags
+from headroom.proxy.tool_schema_savings_policy import (
+ net_hook_tool_saving,
+ without_deferral_flags,
+)
if TYPE_CHECKING:
from fastapi import Request
@@ -3393,6 +3396,11 @@ class AnthropicHandlerMixin:
)
_pre_hook_tokens: int | None = None
+ # Tool-definition tokens a turn hook ADDED (e.g. a search tool). Folded
+ # into the forwarded count below so the headline nets them; shrinkage
+ # stays on the turn_hook_tools_saved_tokens tag.
+ _th_tool_growth = 0
+ _th_saved = 0
_req_ctx: TurnContext | None = None
if registered_turn_hooks():
_req_ctx = TurnContext(
@@ -3435,6 +3443,7 @@ class AnthropicHandlerMixin:
else 0
)
_th_saved = max(0, _th_tok_before - _th_tok_after)
+ _th_tool_growth = max(0, _th_tok_after - _th_tok_before)
if _th_saved > 0:
tags["turn_hook_tools_saved_tokens"] = (
int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved
@@ -3560,6 +3569,8 @@ class AnthropicHandlerMixin:
if 0 < _tool_tokens_after < _tool_tokens_before:
original_tokens += _tool_tokens_before
optimized_tokens += _tool_tokens_after
+ # A tool a turn hook added is sent too: it reduces the saving.
+ optimized_tokens += _th_tool_growth
# First-appearance accounting for matured Reads. The client
# re-sends the raw conversation every turn, so this diff would
# otherwise re-book a matured Read's removal on every request
@@ -3580,7 +3591,11 @@ class AnthropicHandlerMixin:
# Advisory, like the maturation pass itself: a failure
# here must not skip the recount around it.
logger.debug("maturation replay debt skipped", exc_info=True)
- tokens_saved = max(0, original_tokens - optimized_tokens - _replay_debt)
+ # Signed, then clamped once: message text a hook added offsets
+ # the tool definitions it removed (booked on the tag above).
+ tokens_saved = net_hook_tool_saving(
+ original_tokens - optimized_tokens - _replay_debt, tags, _th_saved
+ )
# Attribute the fold to the hook ONLY when the hook itself reduced
# tokens (same-tokenizer pre vs post) — not when the recount above
# merely normalized a cross-estimator scale difference.
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index f649553d5..64861b0df 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -45,7 +45,10 @@ from headroom.proxy.provider_usage import (
from headroom.proxy.rate_limit_identity import rate_limit_identity
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
-from headroom.proxy.tool_schema_savings_policy import without_deferral_flags
+from headroom.proxy.tool_schema_savings_policy import (
+ net_hook_tool_saving,
+ without_deferral_flags,
+)
from headroom.proxy.upstream_guard import is_safe_upstream_url
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
from headroom.proxy.ws_session_registry import (
@@ -4346,6 +4349,7 @@ class OpenAIHandlerMixin:
)
_th_ctx: TurnContext | None = None
+ _th_raw_saved = None
if registered_turn_hooks():
_th_tools_before = body.get("tools")
_th_tok_before = (
@@ -4393,7 +4397,17 @@ class OpenAIHandlerMixin:
optimized_tokens = _th_msg_after
if 0 < tool_tokens_after_compaction < tool_tokens_before_compaction:
optimized_tokens += tool_tokens_after_compaction
- tokens_saved = max(0, original_tokens - optimized_tokens)
+ # A tool the hook added is sent too: it reduces the saving.
+ _th_tools_after_count = (
+ tokenizer.count_text(
+ json.dumps(without_deferral_flags(_th_ctx.tools), default=str)
+ )
+ if _th_ctx.tools
+ else 0
+ )
+ optimized_tokens += max(0, _th_tools_after_count - _th_tok_before)
+ _th_raw_saved: int | None = original_tokens - optimized_tokens
+ tokens_saved = max(0, _th_raw_saved)
# Attribute to the hook ONLY when the hook itself reduced tokens.
if _th_msg_before is not None and _th_msg_after < _th_msg_before:
transforms_applied.append("turn_hook")
@@ -4410,6 +4424,10 @@ class OpenAIHandlerMixin:
int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved
)
transforms_applied.append(f"turn_hook:tools:{_th_saved}tok")
+ # Signed, then clamped once: message text a hook added offsets the
+ # tool definitions it removed.
+ if _th_raw_saved is not None:
+ tokens_saved = net_hook_tool_saving(_th_raw_saved, tags, _th_saved)
# Compatibility shim: GPT-5 / o-series chat models REJECT the legacy
# `max_tokens` ("Unsupported parameter … Use 'max_completion_tokens'
@@ -10462,6 +10480,11 @@ class OpenAIHandlerMixin:
if _turn is not None:
_turn.prepare(model_name=model_name, tags=tags, config=self.config)
+ # Whether the returned ``tokens_after`` was counted over the FINAL,
+ # post-hook messages (so it already includes any text a hook added).
+ # Set by whichever closure runs; read after the executor.
+ _after_counts_hook = [False]
+
def _run_stateless():
result = pipeline.apply(messages=messages, model=model, **pipeline_kwargs)
final = result.messages
@@ -10470,6 +10493,7 @@ class OpenAIHandlerMixin:
final = _turn.transform(final)
if _turn.folded_messages:
tokens_after = _turn.count_messages(final, tokens_after)
+ _after_counts_hook[0] = True
return (
result,
final,
@@ -10559,6 +10583,7 @@ class OpenAIHandlerMixin:
_tok = get_tokenizer(model_name)
raw_tokens_before = _tok.count_messages(messages)
final_tokens_after = _tok.count_messages(final)
+ _after_counts_hook[0] = True
except Exception as e:
# Fail-open, but LOUD: this fallback reverts to the
# pipeline's counts of the cache-swapped input, which
@@ -10609,7 +10634,19 @@ class OpenAIHandlerMixin:
ccr_hashes = _response_ccr_hashes(final_messages, result.markers_inserted)
- tokens_saved = max(0, tokens_before - tokens_after)
+ # Tools and message text a turn hook added are sent too: they reduce
+ # the saving, and offset tools the hook removed (signed, clamped once).
+ _hook_tool_saved = 0
+ if _turn is not None:
+ tokens_after += max(0, int(getattr(_turn, "tool_growth_tokens", 0) or 0))
+ # Only a count of pre-hook messages lacks the text a hook added;
+ # a recount of the final messages (stateless fold, session) has it.
+ if not _after_counts_hook[0]:
+ tokens_after += max(0, int(getattr(_turn, "message_growth_tokens", 0) or 0))
+ _hook_tool_saved = int(getattr(_turn, "hook_tool_saved_tokens", 0) or 0)
+ tokens_saved = net_hook_tool_saving(
+ tokens_before - tokens_after, tags, _hook_tool_saved
+ )
latency_ms = (time.time() - start_time) * 1000
_transforms_applied = list(result.transforms_applied or ())
# The claimed turn adds its labels and response fields BEFORE the
diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.py
index c75f09e94..c77469d17 100644
--- a/headroom/proxy/tool_schema_savings_policy.py
+++ b/headroom/proxy/tool_schema_savings_policy.py
@@ -183,6 +183,26 @@ def tool_schema_saved_from_tags(tags: object) -> int:
return total
+def net_hook_tool_saving(raw_saved: int, tags: object, hook_tool_saved: int) -> int:
+ """Net a request's message-side saving against the tool saving its turn
+ hooks booked on ``turn_hook_tools_saved_tokens``; return it clamped at 0.
+
+ The headline is ``tokens_saved`` (clamped) plus that tag. A hook that adds
+ message text while removing tools leaves ``raw_saved`` negative, and the
+ clamp alone would drop the growth while the tag kept the full removal. The
+ deficit is debited from the tag first (at most what the hooks booked on
+ this request), so headline == messages + tools, signed, clamped once.
+ """
+ raw = int(raw_saved)
+ booked = max(0, int(hook_tool_saved or 0))
+ if raw < 0 and booked > 0 and isinstance(tags, dict):
+ offset = min(-raw, booked)
+ current = int(tags.get("turn_hook_tools_saved_tokens", 0) or 0)
+ tags["turn_hook_tools_saved_tokens"] = max(0, current - offset)
+ raw += offset
+ return max(0, raw)
+
+
def headline_tokens_saved(tokens_saved: object, tags: object) -> int:
"""Return the single "Tokens saved" figure for one request.
diff --git a/headroom/proxy/turn_hooks.py b/headroom/proxy/turn_hooks.py
index 6e4987075..5d9f6e228 100644
--- a/headroom/proxy/turn_hooks.py
+++ b/headroom/proxy/turn_hooks.py
@@ -207,6 +207,11 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> No
# Deferral booked before or by a hook; reconciled once every hook has run,
# since a later hook may un-defer booked tools (tool search's hot tools).
booking = deferred_booking(ctx.tags, ctx.tools)
+ # Positive nets wait until every hook has run: a later hook that grows the
+ # request (adds a tool) offsets earlier savings, so the ledger never credits
+ # more than the whole sequence actually removed.
+ gains: list[tuple[str, int, dict[str, int]]] = []
+ sequence_net = 0
for hook in registered_turn_hooks():
if stream_safe_only and not getattr(hook, "stream_safe", False):
continue
@@ -221,24 +226,28 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> No
if ctx.count_tools is not None:
before_tools = ctx.count_tools(without_deferral_flags(ctx.tools))
fn(ctx)
+ # Signed deltas, netted: a hook that folds messages but ADDS a tool
+ # (skill search's ``search_skills``) saved the difference, not the
+ # fold alone. Booked only when the net is a saving.
message_saved = (
- max(0, before_messages - ctx.count_messages(ctx.messages))
+ before_messages - ctx.count_messages(ctx.messages)
if before_messages is not None and ctx.count_messages is not None
else 0
)
tool_saved = (
- max(0, before_tools - ctx.count_tools(without_deferral_flags(ctx.tools)))
+ before_tools - ctx.count_tools(without_deferral_flags(ctx.tools))
if before_tools is not None and ctx.count_tools is not None
else 0
)
- if message_saved or tool_saved:
- ctx.record_savings(
- getattr(hook, "savings_source", getattr(hook, "name", type(hook).__name__)),
- tokens=message_saved + tool_saved,
- details={
- "message_tokens_saved": message_saved,
- "tool_tokens_saved": tool_saved,
- },
+ net_saved = message_saved + tool_saved
+ sequence_net += net_saved
+ if net_saved > 0:
+ gains.append(
+ (
+ getattr(hook, "savings_source", getattr(hook, "name", type(hook).__name__)),
+ net_saved,
+ {"message_tokens_saved": message_saved, "tool_tokens_saved": tool_saved},
+ )
)
except Exception: # a hook must never break the proxy
log.exception("turn hook %r on_request failed", getattr(hook, "name", hook))
@@ -246,6 +255,29 @@ def run_request_hooks(ctx: TurnContext, *, stream_safe_only: bool = False) -> No
# This hook booked (or re-booked) the deferral: it is the new baseline.
booking = deferred_booking(ctx.tags, ctx.tools)
reconcile_deferred_tokens(ctx.tags, booking, ctx.tools, ctx.count_tools)
+ # Book gains in run order up to the sequence's net saving; a loss that
+ # exceeds them all books nothing (never a negative entry).
+ budget = max(0, sequence_net)
+ for source, net_saved, details in gains:
+ tokens = min(net_saved, budget)
+ if tokens <= 0:
+ break
+ budget -= tokens
+ if tokens < net_saved:
+ # Later hooks' cost comes off this credit; share it across the
+ # message and tool parts so the breakdown still sums to ``tokens``.
+ # The uncapped deltas stay for diagnosis.
+ message_part = round(details["message_tokens_saved"] * tokens / net_saved)
+ details = {
+ "message_tokens_saved": message_part,
+ "tool_tokens_saved": tokens - message_part,
+ "uncapped_message_tokens_saved": details["message_tokens_saved"],
+ "uncapped_tool_tokens_saved": details["tool_tokens_saved"],
+ }
+ try:
+ ctx.record_savings(source, tokens=tokens, details=details)
+ except Exception:
+ log.exception("turn hook savings for %r not recorded", source)
async def run_response_hooks(
diff --git a/tests/gateway/test_hook_tool_growth.py b/tests/gateway/test_hook_tool_growth.py
new file mode 100644
index 000000000..cc91ebf50
--- /dev/null
+++ b/tests/gateway/test_hook_tool_growth.py
@@ -0,0 +1,124 @@
+"""/v1/compress nets tool definitions a turn hook adds against its saving.
+
+A hook that folds messages but adds a tool (skill search's search tool) sends
+those tool tokens too; ``tokens_after`` must include them, as on the chat path.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import pytest
+
+from headroom.proxy.savings_attribution import from_tags
+from headroom.proxy.turn_hooks import register_turn_hook
+from tests.gateway.conftest import compress
+
+_LONG = "line of tool output that a hook can fold away entirely " * 60
+_ADDED_TOOL = {
+ "name": "search_skills",
+ "description": "Find a skill by what it does. " * 40,
+ "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}},
+}
+
+
+class _FoldAndAddTool:
+ name = savings_source = "skill_search"
+ stream_safe = True
+
+ def __init__(self, add_tool: bool) -> None:
+ self.add_tool = add_tool
+
+ def on_request(self, ctx: Any) -> None:
+ ctx.messages = [{**m, "content": "folded"} for m in ctx.messages]
+ if self.add_tool:
+ ctx.tools = [*(ctx.tools or []), dict(_ADDED_TOOL)]
+
+
+def _run(make_headroom_client: Any, *, add_tool: bool) -> dict[str, Any]:
+ register_turn_hook(_FoldAndAddTool(add_tool))
+ client = make_headroom_client(optimize=False)
+ body = {
+ "model": "claude-sonnet-4-5",
+ "messages": [{"role": "user", "content": _LONG}],
+ "tools": [{"name": "noop", "description": "x", "input_schema": {"type": "object"}}],
+ "gateway": {"can_redrive": False, "can_relay_response": False},
+ }
+ resp = compress(client, body)
+ assert resp.status_code == 200, resp.text
+ return resp.json()
+
+
+def test_tools_a_hook_adds_count_as_sent(make_headroom_client) -> None:
+ from headroom.proxy.turn_hooks import clear_turn_hooks
+
+ plain = _run(make_headroom_client, add_tool=False)
+ clear_turn_hooks()
+ grown = _run(make_headroom_client, add_tool=True)
+ assert plain["tokens_saved"] > 0
+ added = grown["tokens_after"] - plain["tokens_after"]
+ assert added > 100 # the search tool's definition
+ assert grown["tokens_saved"] == plain["tokens_saved"] - added
+ assert grown["tokens_before"] == plain["tokens_before"]
+
+
+class _AddTextRemoveTools:
+ name = savings_source = "rewriter"
+ stream_safe = True
+
+ def __init__(self, repeat: int) -> None:
+ self.repeat = repeat
+
+ def on_request(self, ctx: Any) -> None:
+ ctx.messages = [*ctx.messages, {"role": "user", "content": "steer. " * self.repeat}]
+ ctx.tools = []
+
+
[email protected]("session", [False, True], ids=["stateless", "session"])
[email protected]("repeat", [5, 2_000], ids=["net-positive", "net-negative"])
+def test_message_growth_offsets_tools_a_hook_removed(
+ make_headroom_client, outcome_spy, repeat, session
+) -> None:
+ """The opposite signed case on /v1/compress: a hook appends message text
+ and drops the tools. The headline is removed tools minus added text,
+ clamped once, not the full tool removal."""
+ from headroom.proxy.tool_schema_savings_policy import headline_tokens_saved
+ from headroom.tokenizers import get_tokenizer
+
+ register_turn_hook(_AddTextRemoveTools(repeat))
+ client = make_headroom_client(optimize=False)
+ outcomes = outcome_spy(client)
+ before = [{"role": "user", "content": "hi"}]
+ body = {
+ "model": "claude-sonnet-4-5",
+ "messages": before,
+ "tools": [dict(_ADDED_TOOL)],
+ "gateway": {"can_redrive": False, "can_relay_response": False},
+ }
+ if session:
+ # Session turns recount the final (post-hook) messages: the added
+ # text must not be counted a second time on top of that.
+ body["config"] = {"session_id": f"growth-{repeat}"}
+ resp = compress(client, body)
+ assert resp.status_code == 200, resp.text
+ tok = get_tokenizer("claude-sonnet-4-5")
+ growth = tok.count_messages(
+ [*before, {"role": "user", "content": "steer. " * repeat}]
+ ) - tok.count_messages(before)
+ removed = tok.count_text(json.dumps([_ADDED_TOOL], default=str))
+ net = removed - growth
+ assert (net > 0) == (repeat == 5)
+ outcome = outcomes[-1]
+ headline = headline_tokens_saved(outcome.tokens_saved, outcome.tags)
+ # The gateway compacts tool schemas before hooks run, so compare with the
+ # hook's own measured deltas (the runner's ledger), not the raw schema.
+ entries = from_tags(outcome.tags)
+ if net > 0:
+ (entry,) = entries
+ hook_net = entry["details"]["tool_tokens_saved"] + entry["details"]["message_tokens_saved"]
+ assert entry["details"]["message_tokens_saved"] < 0
+ assert headline == hook_net == entry["tokens"]
+ else:
+ assert entries == []
+ assert headline == 0
diff --git a/tests/test_hook_net_tool_additions.py b/tests/test_hook_net_tool_additions.py
new file mode 100644
index 000000000..0ff8107cb
--- /dev/null
+++ b/tests/test_hook_net_tool_additions.py
@@ -0,0 +1,388 @@
+"""A turn hook that folds messages but ADDS a tool saved the difference.
+
+headroom-skill-search rewrites the skill list in the messages and appends a
+``search_skills`` tool. The runner used to clamp the message and tool halves
+separately, so the added tool never reduced the booked saving, and the
+handlers left it out of the forwarded count, so the headline overstated too.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import httpx
+import pytest
+import respx
+
+from headroom.proxy.savings_attribution import from_tags
+from headroom.proxy.turn_hooks import (
+ TurnContext,
+ clear_turn_hooks,
+ register_turn_hook,
+ run_request_hooks,
+)
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.proxy.loopback_guard import require_loopback # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+from headroom.tokenizers import get_tokenizer # noqa: E402
+
+SEARCH_TOOL = {
+ "name": "search_skills",
+ "description": "Search the skill catalogue by keyword. " * 10,
+ "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
+}
+
+
[email protected](autouse=True)
+def _clean_registry():
+ clear_turn_hooks()
+ yield
+ clear_turn_hooks()
+
+
+class _FoldAndAddTool:
+ """Shaped like SkillSearchHook: shrink a message, append a search tool."""
+
+ name = savings_source = "skill_search"
+ stream_safe = True
+
+ def __init__(self, folded_text: str) -> None:
+ self.folded_text = folded_text
+
+ def on_request(self, ctx: TurnContext) -> None:
+ ctx.messages = [{"role": "user", "content": self.folded_text}]
+ ctx.tools = [*(ctx.tools or []), SEARCH_TOOL]
+
+
+def _count(value: Any) -> int:
+ return len(json.dumps(value, default=str)) // 4 if value else 0
+
+
+def _run(hook: Any, text: str) -> TurnContext:
+ register_turn_hook(hook)
+ ctx = TurnContext(
+ provider="anthropic",
+ model="claude-sonnet-4-6",
+ messages=[{"role": "user", "content": text}],
+ tools=[],
+ count_messages=_count,
+ count_tools=_count,
+ )
+ run_request_hooks(ctx)
+ return ctx
+
+
+def test_ledger_entry_is_net_of_the_added_tool() -> None:
+ long_text = "skill line\n" * 400
+ ctx = _run(_FoldAndAddTool("short"), long_text)
+ (entry,) = from_tags(ctx.tags)
+ folded = _count([{"role": "user", "content": long_text}]) - _count(
+ [{"role": "user", "content": "short"}]
+ )
+ added = _count([SEARCH_TOOL])
+ assert entry["tokens"] == folded - added
+ assert entry["details"]["tool_tokens_saved"] == -added
+
+
+def test_no_entry_when_the_added_tool_outweighs_the_fold() -> None:
+ ctx = _run(_FoldAndAddTool("short"), "a bit longer than short")
+ assert from_tags(ctx.tags) == []
+
+
+def test_anthropic_headline_nets_a_tool_the_hook_added(monkeypatch) -> None:
+ long_text = "skill line number one\n" * 300
+ register_turn_hook(_FoldAndAddTool("short"))
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ outcomes: list[Any] = []
+
+ async def _spy(_self, outcome, *a, **kw): # noqa: ANN001, ANN002, ANN003, ANN202
+ outcomes.append(outcome)
+
+ monkeypatch.setattr(type(app.state.proxy), "_record_request_outcome", _spy, raising=True)
+ respx.post("https://api.anthropic.com/v1/messages").mock(
+ return_value=httpx.Response(
+ 200,
+ json={
+ "id": "a",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 100, "output_tokens": 1},
+ },
+ )
+ )
+ with TestClient(app) as client:
+ result = client.post(
+ "/v1/messages",
+ json={
+ "model": "claude-sonnet-4-5",
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": long_text}],
+ },
+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
+ )
+ assert result.status_code == 200
+ tok = get_tokenizer("claude-sonnet-4-5")
+ folded = tok.count_messages([{"role": "user", "content": long_text}]) - tok.count_messages(
+ [{"role": "user", "content": "short"}]
+ )
+ added = tok.count_text(json.dumps([SEARCH_TOOL], default=str))
+ assert folded > added > 0
+ assert outcomes[-1].tokens_saved == folded - added
+
+
+class _FoldAndAddFunction(_FoldAndAddTool):
+ def on_request(self, ctx: TurnContext) -> None:
+ ctx.messages = [{"role": "user", "content": self.folded_text}]
+ ctx.tools = [*(ctx.tools or []), {"type": "function", "function": SEARCH_TOOL}]
+
+
+def test_openai_chat_headline_nets_a_tool_the_hook_added(monkeypatch) -> None:
+ long_text = "skill line number one\n" * 300
+ register_turn_hook(_FoldAndAddFunction("short"))
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ outcomes: list[Any] = []
+
+ async def _spy(_self, outcome, *a, **kw): # noqa: ANN001, ANN002, ANN003, ANN202
+ outcomes.append(outcome)
+
+ monkeypatch.setattr(type(app.state.proxy), "_record_request_outcome", _spy, raising=True)
+ respx.post("https://api.openai.com/v1/chat/completions").mock(
+ return_value=httpx.Response(
+ 200,
+ json={
+ "id": "a",
+ "choices": [
+ {"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
+ ],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 1},
+ },
+ )
+ )
+ with TestClient(app) as client:
+ result = client.post(
+ "/v1/chat/completions",
+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": long_text}]},
+ headers={"authorization": "Bearer sk-test"},
+ )
+ assert result.status_code == 200
+ tok = get_tokenizer("gpt-4o")
+ folded = tok.count_messages([{"role": "user", "content": long_text}]) - tok.count_messages(
+ [{"role": "user", "content": "short"}]
+ )
+ added = tok.count_text(json.dumps([{"type": "function", "function": SEARCH_TOOL}], default=str))
+ assert folded > added > 0
+ assert outcomes[-1].tokens_saved == folded - added
+
+
+# ── a later hook's loss offsets an earlier hook's gain ───────────────────────
+
+
+class _Fold:
+ name = savings_source = "fold"
+ stream_safe = True
+
+ def on_request(self, ctx: TurnContext) -> None:
+ ctx.messages = [{"role": "user", "content": "x"}]
+
+
+class _AddBigTool:
+ name = savings_source = "adder"
+ stream_safe = True
+
+ def __init__(self, size: int) -> None:
+ self.size = size
+
+ def on_request(self, ctx: TurnContext) -> None:
+ ctx.tools = [*(ctx.tools or []), {"name": "t", "description": "d" * self.size}]
+
+
+def _sequence(*hooks: object) -> tuple[list[dict], int]:
+ for hook in hooks:
+ register_turn_hook(hook)
+ count = lambda v: len(json.dumps(v, default=str)) // 4 if v else 0 # noqa: E731
+ messages = [{"role": "user", "content": "m" * 800}]
+ ctx = TurnContext(
+ provider="anthropic",
+ model="claude-sonnet-4-6",
+ messages=messages,
+ tools=[],
+ count_messages=count,
+ count_tools=count,
+ )
+ before = count(messages)
+ run_request_hooks(ctx)
+ net = before - count(ctx.messages) - count(ctx.tools)
+ return from_tags(ctx.tags), net
+
+
+def test_a_later_hook_that_grows_more_than_was_saved_cancels_the_credit() -> None:
+ entries, net = _sequence(_Fold(), _AddBigTool(2_000))
+ assert net < 0
+ assert entries == []
+
+
+def test_a_later_partial_loss_caps_the_earlier_credit_at_the_net() -> None:
+ entries, net = _sequence(_Fold(), _AddBigTool(200))
+ assert net > 0
+ assert [(e["source"], e["tokens"]) for e in entries] == [("fold", net)]
+ (entry,) = entries
+ details = entry["details"]
+ # The breakdown sums to the credited amount; the raw deltas are kept.
+ assert details["message_tokens_saved"] + details["tool_tokens_saved"] == net
+ assert details["uncapped_message_tokens_saved"] > net
+ assert details["uncapped_tool_tokens_saved"] == 0
+
+
+# ── message growth offsets tool removal in the handler headline ──────────────
+
+BIG_TOOL = {
+ "name": "big_tool",
+ "description": "Does many things in many ways. " * 40,
+ "input_schema": {"type": "object"},
+}
+
+
+class _AddTextRemoveTools:
+ """The opposite signed case: append message text, drop every tool."""
+
+ name = savings_source = "rewriter"
+ stream_safe = True
+
+ def __init__(self, added_text: str) -> None:
+ self.added_text = added_text
+
+ def on_request(self, ctx: TurnContext) -> None:
+ ctx.messages = [*ctx.messages, {"role": "user", "content": self.added_text}]
+ ctx.tools = []
+
+
+def _app_with_spy(monkeypatch) -> tuple[Any, list[Any]]:
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ outcomes: list[Any] = []
+
+ async def _spy(_self, outcome, *a, **kw): # noqa: ANN001, ANN002, ANN003, ANN202
+ outcomes.append(outcome)
+
+ monkeypatch.setattr(type(app.state.proxy), "_record_request_outcome", _spy, raising=True)
+ return app, outcomes
+
+
+def _expected(model: str, before: list, added_text: str, tools: list) -> int:
+ tok = get_tokenizer(model)
+ grown = tok.count_messages([*before, {"role": "user", "content": added_text}])
+ message_growth = grown - tok.count_messages(before)
+ removed = tok.count_text(json.dumps(tools, default=str))
+ return removed - message_growth
+
+
[email protected]("repeat", [5, 400], ids=["net-positive", "net-negative"])
+def test_anthropic_headline_nets_added_text_against_removed_tools(monkeypatch, repeat) -> None:
+ from headroom.proxy.tool_schema_savings_policy import headline_tokens_saved
+
+ added = "extra steering text. " * repeat
+ register_turn_hook(_AddTextRemoveTools(added))
+ app, outcomes = _app_with_spy(monkeypatch)
+ respx.post("https://api.anthropic.com/v1/messages").mock(
+ return_value=httpx.Response(
+ 200,
+ json={
+ "id": "a",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 100, "output_tokens": 1},
+ },
+ )
+ )
+ before = [{"role": "user", "content": "hi"}]
+ with TestClient(app) as client:
+ result = client.post(
+ "/v1/messages",
+ json={
+ "model": "claude-sonnet-4-5",
+ "max_tokens": 16,
+ "messages": before,
+ "tools": [BIG_TOOL],
+ },
+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
+ )
+ assert result.status_code == 200
+ net = _expected("claude-sonnet-4-5", before, added, [BIG_TOOL])
+ assert (net > 0) == (repeat == 5)
+ outcome = outcomes[-1]
+ assert headline_tokens_saved(outcome.tokens_saved, outcome.tags) == max(0, net)
+
+
[email protected]("repeat", [5, 400], ids=["net-positive", "net-negative"])
+def test_openai_chat_headline_nets_added_text_against_removed_tools(monkeypatch, repeat) -> None:
+ from headroom.proxy.tool_schema_savings_policy import headline_tokens_saved
+
+ added = "extra steering text. " * repeat
+ register_turn_hook(_AddTextRemoveTools(added))
+ app, outcomes = _app_with_spy(monkeypatch)
+ respx.post("https://api.openai.com/v1/chat/completions").mock(
+ return_value=httpx.Response(
+ 200,
+ json={
+ "id": "a",
+ "choices": [
+ {"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
+ ],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 1},
+ },
+ )
+ )
+ tools = [{"type": "function", "function": BIG_TOOL}]
+ before = [{"role": "user", "content": "hi"}]
+ with TestClient(app) as client:
+ result = client.post(
+ "/v1/chat/completions",
+ json={"model": "gpt-4o", "messages": before, "tools": tools},
+ headers={"authorization": "Bearer sk-test"},
+ )
+ assert result.status_code == 200
+ net = _expected("gpt-4o", before, added, tools)
+ assert (net > 0) == (repeat == 5)
+ outcome = outcomes[-1]
+ assert headline_tokens_saved(outcome.tokens_saved, outcome.tags) == max(0, net)