Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(stats): report one "Tokens Saved" headline across every harness (#2737)
Tejas Chopra8262a4a3217b
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.htmlindex da86d332a..fe9b4562a 100644--- a/headroom/dashboard/templates/dashboard.html+++ b/headroom/dashboard/templates/dashboard.html@@ -209,7 +209,7 @@<div class="grid grid-cols-1 gap-4 mb-6 md:grid-cols-2 lg:grid-cols-3"><!-- Token Savings (%) --><div class="bg-surface rounded-lg p-4 border border-border">- <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Token Savings</div>+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tokens Saved</div><div class="flex items-baseline gap-2"><span class="text-3xl font-light tabular-nums text-accent" x-text="formatNumber(stats.tokens?.saved || 0)"></span><!-- Active compression ratio: savings as a fraction of tokens we@@ -270,10 +270,13 @@</template></div>- <!-- Tool-Schema Deferral (tool search) — only rendered when there's a saving to show -->+ <!-- Tool-schema deferral: a COMPONENT of the Tokens Saved headline above+ (stats.tokens.saved is all-layers), not a rival metric. Labelled as+ such so a tool-heavy session doesn't read as "0 saved + some other+ number". Only rendered when there's a saving to show. --><template x-if="(stats.savings?.by_layer?.tool_search?.tokens || 0) > 0"><div class="bg-surface rounded-lg p-4 border border-border">- <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tool-Schema Deferral</div>+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tokens Saved · Tool Schemas</div><div class="flex items-baseline gap-2"><span class="text-3xl font-light tabular-nums text-emerald-400" x-text="formatNumber(stats.savings?.by_layer?.tool_search?.tokens || 0)"></span><span class="text-sm text-gray-400">tokens</span>diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.pyindex 59fe4c0dd..4deec44ac 100644--- a/headroom/perf/analyzer.py+++ b/headroom/perf/analyzer.py@@ -483,16 +483,25 @@ def format_report(report: PerfReport) -> str:total_after = sum(r.tokens_after for r in records)total_saved = sum(r.tokens_saved for r in records)total_tool_saved = sum(r.tool_saved for r in records)+ total_headline_saved = total_saved + total_tool_savedpct = (total_saved / total_before * 100) if total_before > 0 else 0+ # All-layers denominator: deferred tool schemas were never in tok_before (they+ # don't reach count_messages), so the pre-Headroom world is tok_before + them.+ # Same construction the proxy's /api/stats uses for total_before_compression —+ # the headline number and the headline percent must share a numerator, or the+ # tile reads "60,920 saved (0.1%)" off two different definitions of saved.+ headline_before = total_before + total_tool_saved+ headline_pct = (total_headline_saved / headline_before * 100) if headline_before > 0 else 0lines.append(f"Requests: {len(records)}")- lines.append(f"Tokens: {total_before:,} -> {total_after:,} ({pct:.1f}% reduction)")- lines.append(f"Total saved: {total_saved:,} tokens (messages)")- # Tool-schema savings (deferral + turn-hook tool shrink) are counted apart- # from message compression — messages never include tool bytes — so surface- # them explicitly instead of hiding a tool-heavy turn's win behind tok_saved=0.+ lines.append(f"Tokens: {total_before:,} -> {total_after:,} ({pct:.1f}% messages)")+ # ONE headline. Tool-schema deferral can't move tok_before/after (messages never+ # include tool bytes), so it used to render as a rival "Tool saved" line — which+ # read as a side metric and hid the win on tool-heavy turns where tok_saved=0.+ lines.append(f"Tokens saved: {total_headline_saved:,} ({headline_pct:.1f}% reduction)")if total_tool_saved > 0:- lines.append(f"Tool saved: {total_tool_saved:,} tokens (tool schemas, deferral)")+ lines.append(f" · messages {max(0, total_saved):,}")+ lines.append(f" · tool schemas {total_tool_saved:,}")lines.append("")# Per-model breakdown with list prices@@ -978,6 +987,7 @@ def build_perf_summary(report: PerfReport) -> dict:total_after = sum(r.tokens_after for r in records)total_saved = sum(r.tokens_saved for r in records)total_tool_saved = sum(r.tool_saved for r in records)+ total_headline_saved = total_saved + total_tool_savedtotal_cr = sum(r.cache_read for r in records)total_cw = sum(r.cache_write for r in records)@@ -1033,6 +1043,11 @@ def build_perf_summary(report: PerfReport) -> dict:"total_requests": len(records),"total_tokens_before": total_before,"total_tokens_after": total_after,+ # total_tokens_saved is the headline (messages + tool-schema deferral); the two+ # components stay for consumers that break the number down. See+ # headroom.proxy.tool_schema_savings_policy.+ "total_tokens_saved": total_headline_saved,+ "total_savings_pct": _pct(total_headline_saved, total_before + total_tool_saved),"tokens_saved": total_saved,"tool_saved": total_tool_saved,"savings_pct": _pct(total_saved, total_before),diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex cafd72edd..ee5a9092b 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -2277,14 +2277,32 @@ class AnthropicHandlerMixin:# after tools are finalised (sorting, CCR injection) but before# the PRE_SEND pipeline event so extensions see the compacted# schema. Mirrors the same pass that the OpenAI handler applies.+ #+ # Token accounting (see tool_schema_savings_policy): the compaction passes+ # below rewrite the tool array, so both endpoints are countable and the+ # delta is folded into original/optimized_tokens at the final recount.+ # Without this the savings were computed, debug-logged, and discarded —+ # Claude Code reported them nowhere.+ _tool_tokens_before = 0+ _tool_tokens_after = 0++ def _count_tool_tokens(value: object) -> int:+ try:+ return tokenizer.count_text(json.dumps(value, default=str))+ except Exception:+ return 0+_tools_compaction_started = time.time()try:from headroom.proxy.tool_schema_compaction import compact_tools+ _pre_compaction_tools = body.get("tools")body, _tools_modified, _tools_before_bytes, _tools_after_bytes = compact_tools(body)if _tools_modified:tools = body["tools"]transforms_applied.append("anthropic:tool_schema_compaction")+ _tool_tokens_before = _count_tool_tokens(_pre_compaction_tools)+ _tool_tokens_after = _count_tool_tokens(tools)_tools_compaction_ms = (time.time() - _tools_compaction_started) * 1000logger.debug("[%s] tool schema compaction: %d -> %d bytes (%.0f%% saved) in %.1fms",@@ -2311,12 +2329,18 @@ class AnthropicHandlerMixin:_desc_max = tool_desc_max_chars()if _desc_max > 0:+ _pre_desc_tools = body.get("tools")body, _desc_modified, _desc_before, _desc_after = compact_tool_descriptions(body, _desc_max)if _desc_modified:tools = body["tools"]transforms_applied.append("anthropic:tool_desc_compaction")+ # Runs after schema compaction, so only seed "before" when that+ # pass didn't already; "after" always tracks the latest tools.+ if not _tool_tokens_before:+ _tool_tokens_before = _count_tool_tokens(_pre_desc_tools)+ _tool_tokens_after = _count_tool_tokens(tools)logger.debug("[%s] tool description compaction: %d -> %d bytes (%.0f%% saved, max_chars=%d)",request_id,@@ -2466,6 +2490,8 @@ class AnthropicHandlerMixin:_pre_hook_tokens = tokenizer.count_messages(optimized_messages)except Exception:_pre_hook_tokens = None+ _th_tools_before = body.get("tools")+ _th_tok_before = _count_tool_tokens(_th_tools_before) if _th_tools_before else 0run_request_hooks(_req_ctx)if _req_ctx.messages is not optimized_messages:optimized_messages = _req_ctx.messages@@ -2473,6 +2499,16 @@ class AnthropicHandlerMixin:if _req_ctx.tools is not body.get("tools"):tools = _req_ctx.toolsbody["tools"] = tools+ # A hook may shrink the tool array either by replacing it or in place,+ # so measure the FINAL tools object. Deferral-shaped (removes schemas+ # count_messages never saw), hence a tag rather than a fold — mirrors+ # the OpenAI chat path so a turn-hook extension is credited on both.+ _th_tok_after = _count_tool_tokens(_req_ctx.tools) if _req_ctx.tools else 0+ _th_saved = max(0, _th_tok_before - _th_tok_after)+ if _th_saved > 0:+ tags["turn_hook_tools_saved_tokens"] = (+ int(tags.get("turn_hook_tools_saved_tokens", 0) or 0) + _th_saved+ )# Consistency: report tok_before/tok_after with ONE tokenizer. The pipeline# and the handler use different token estimators, and cache-mode branches@@ -2486,6 +2522,13 @@ class AnthropicHandlerMixin:_orig_snapshot = original_client_messages # noqa: F821 (bound at request start)original_tokens = tokenizer.count_messages(_orig_snapshot)optimized_tokens = tokenizer.count_messages(optimized_messages)+ # Fold the tool-schema/desc compaction delta into BOTH endpoints so+ # tok_before - tok_after == tok_saved stays coherent in the PERF line+ # (count_messages never sees tool bytes). Same shape as the OpenAI chat+ # handler; guarded so a no-op or inflating pass contributes nothing.+ if 0 < _tool_tokens_after < _tool_tokens_before:+ original_tokens += _tool_tokens_before+ optimized_tokens += _tool_tokens_aftertokens_saved = max(0, original_tokens - optimized_tokens)# Attribute the fold to the hook ONLY when the hook itself reduced# tokens (same-tokenizer pre vs post) — not when the recount abovediff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 3c92b4c9b..3a4dbde96 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -30,6 +30,11 @@ from dataclasses import dataclass, fieldfrom datetime import datetimefrom typing import Any+from headroom.proxy.tool_schema_savings_policy import (+ headline_tokens_saved,+ tool_schema_saved_from_tags,+)+logger = logging.getLogger("headroom.proxy")@@ -410,10 +415,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:# Tool-schema savings (deferral + turn-hook tool shrink) live in per-request# tags and never move tok_before/after; aggregate them into Metrics so the# session summary / cost summary / all-layers total can surface the layer.- _otags = outcome.tags or {}- tool_search_saved = int(_otags.get("tool_search_deferred_tokens", 0) or 0) + int(- _otags.get("turn_hook_tools_saved_tokens", 0) or 0- )+ tool_search_saved = tool_schema_saved_from_tags(outcome.tags or {})# 1. Prometheus / SavingsTracker.await handler.metrics.record_request(@@ -498,21 +500,20 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:# line unchanged, and gives ``headroom perf --client X``# parsers a clean key to filter on.client_part = f" client={outcome.client}" if outcome.client else ""- # Tool-schema savings are tracked separately from message compression: tool- # deferral (defer_loading) and turn-hook tool shrink don't move tok_before/after- # (those count messages only), so a tool-heavy turn shows tok_saved=0 while- # genuinely saving thousands of tool-schema tokens. Surface it as its own field- # so `headroom perf` / log readers see the whole picture.- _tags = outcome.tags or {}- tool_saved = int(_tags.get("tool_search_deferred_tokens", 0) or 0) + int(- _tags.get("turn_hook_tools_saved_tokens", 0) or 0- )+ # Tool-schema DEFERRAL savings can't move tok_before/after (those count messages+ # only), so a tool-heavy turn shows tok_saved=0 while genuinely saving thousands of+ # tool-definition tokens. `tool_saved` carries that component and `total_saved` is+ # the sum every user-facing surface reports — see tool_schema_savings_policy for why+ # compaction is already inside tok_saved and must not be added twice.+ tool_saved = tool_schema_saved_from_tags(outcome.tags or {})+ total_saved = headline_tokens_saved(outcome.tokens_saved, outcome.tags or {})logger.info(f"[{outcome.request_id}] PERF "f"model={outcome.model} msgs={outcome.num_messages} "f"tok_before={outcome.original_tokens} tok_after={outcome.optimized_tokens} "f"tok_saved={outcome.tokens_saved} "f"tool_saved={tool_saved} "+ f"total_saved={total_saved} "f"cache_read={outcome.cache_read_tokens} cache_write={outcome.cache_write_tokens} "f"cache_hit_pct={outcome.cache_hit_pct} "f"opt_ms={outcome.overhead_ms:.0f} "diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex f2d47381e..a9ac18caa 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1917,11 +1917,13 @@ class HeadroomProxy(logger.info(f"Failed: {m.requests_failed}")logger.info(f"Input tokens: {m.tokens_input_total:,}")logger.info(f"Output tokens: {m.tokens_output_total:,}")- logger.info(f"Tokens saved: {m.tokens_saved_total:,}")+ # ONE headline: message compression + tool-schema deferral. Deferral can't move+ # tok_before/after (tool bytes never reach count_messages), so it used to print+ # as a separate line that read like a side metric rather than savings.+ logger.info(f"Tokens saved: {m.tokens_saved_total + m.tool_search_saved_total:,}")if m.tool_search_saved_total > 0:- # Tool-schema deferral / turn-hook tool shrink — counted apart from- # message compression (tool bytes never move tok_before/after).- logger.info(f"Tool schemas deferred: {m.tool_search_saved_total:,}")+ logger.info(f" messages: {m.tokens_saved_total:,}")+ logger.info(f" tool schemas: {m.tool_search_saved_total:,}")# Active-compression ratio: savings as a fraction of what we# *attempted* to compress (extracted units + tool schema),# NOT the whole request. The full-request denominator is@@ -3887,9 +3889,18 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# should show — it answers "are we doing well *when we# have something to compress?*" rather than diluting the# win by frozen-prefix bytes we never touched.+ # All-layers numerator, matching denominator: `attempted_input_tokens`+ # already counts tool schemas we COMPACTED, so pairing it with a+ # compression-only numerator undercounted every tool-heavy session.+ # Deferred schemas are added to both sides — they were attempted work+ # that succeeded completely."active_savings_percent": round(- (proxy_compression_tokens / attempted_input_tokens * 100)- if attempted_input_tokens > 0+ (+ all_layers_tokens_saved+ / (attempted_input_tokens + m.tool_search_saved_total)+ * 100+ )+ if (attempted_input_tokens + m.tool_search_saved_total) > 0else 0,2,),diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.pyindex b3ed33598..5827d0c95 100644--- a/headroom/proxy/tool_schema_savings_policy.py+++ b/headroom/proxy/tool_schema_savings_policy.py@@ -1,4 +1,27 @@-"""Tool-schema savings attribution policy for proxy stats."""+"""Tool-schema savings attribution policy for proxy stats.++Headroom saves input tokens in two accounting shapes, and the split is not a+style choice — it follows from what each transform can observe:++* **Compaction** (``*:tool_schema_compaction``, ``*:tool_desc_compaction``)+ rewrites the tool array in place, so both endpoints are countable. Handlers+ fold the delta into ``original_tokens``/``optimized_tokens``, which keeps+ ``tok_before - tok_after == tok_saved`` coherent in the PERF line. It is+ therefore ALREADY inside ``tokens_saved`` and must never be added again.+* **Deferral / hook shrink** (tool search, turn hooks) removes schemas that+ ``count_messages`` never saw, so it cannot move ``original_tokens``. It is+ recorded in per-request tags and is ADDITIVE to ``tokens_saved``.++The one rule a caller needs: the headline is+``tokens_saved + tool_schema_saved_from_tags(tags)``. Use+:func:`headline_tokens_saved` rather than open-coding it — three surfaces had+drifted inline copies of that sum, and two harnesses were silently dropping+their compaction savings entirely because the convention was never written down.++Adding a new tool-schema-shrinking feature? If it moves the tool array, fold it+in the handler like the compaction sites do. If it defers schemas, add its tag+name to :data:`TOOL_SCHEMA_SAVINGS_TAGS` and every surface picks it up.+"""from __future__ import annotations@@ -13,6 +36,8 @@ def tool_schema_saved_from_tags(tags: object) -> int:The summed tags are set only on paths where Headroom performed the deferral,so clients that already had tool search enabled contribute zero here.++ These tags are additive to ``tokens_saved`` — see the module docstring."""if not isinstance(tags, dict):return 0@@ -24,3 +49,23 @@ def tool_schema_saved_from_tags(tags: object) -> int:except (TypeError, ValueError):continuereturn total+++def headline_tokens_saved(tokens_saved: object, tags: object) -> int:+ """Return the single "Tokens saved" figure for one request.++ This is the only correct total to show a user: message compression plus the+ tool-definition tokens that never entered the context. Every reporting+ surface (PERF line, ``headroom perf``, ``/api/stats``, dashboard, session+ summary) must route through here so they cannot disagree.++ Clamped at zero: handlers already revert any inflation before forwarding, so+ a negative is a token-count artifact that never reached the model.+ """+ base = 0+ if isinstance(tokens_saved, (int, float, str)) and not isinstance(tokens_saved, bool):+ try:+ base = int(tokens_saved)+ except (TypeError, ValueError):+ base = 0+ return max(0, base + tool_schema_saved_from_tags(tags))diff --git a/tests/test_tool_schema_savings_policy.py b/tests/test_tool_schema_savings_policy.pyindex 999a7800e..a44c9b124 100644--- a/tests/test_tool_schema_savings_policy.py+++ b/tests/test_tool_schema_savings_policy.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom headroom.proxy.tool_schema_savings_policy import (TOOL_SCHEMA_SAVINGS_TAGS,+ headline_tokens_saved,tool_schema_saved_from_tags,)@@ -41,3 +42,101 @@ def test_tool_schema_savings_tags_are_stable() -> None:"tool_search_deferred_tokens","turn_hook_tools_saved_tokens",)+++# ── headline_tokens_saved: the one figure every surface reports ────────────────+# Headroom saves tool-definition tokens in two accounting shapes — compaction+# folds into tokens_saved, deferral is tagged and additive. Both existed before+# but the rule was never written down, so two harnesses dropped their compaction+# savings and three surfaces open-coded the sum. These cases pin the contract.+++def test_folded_compaction_is_not_counted_twice() -> None:+ """Compaction is ALREADY inside tokens_saved (handlers fold both endpoints).++ Adding an attribution amount back on top would inflate every tool-heavy turn.+ """+ assert headline_tokens_saved(420, {}) == 420+++def test_deferral_tags_are_additive_to_tokens_saved() -> None:+ """Deferral removes schemas count_messages never saw, so it can't be folded."""+ tags = {"tool_search_deferred_tokens": 9639}+ assert headline_tokens_saved(0, tags) == 9639+ assert headline_tokens_saved(1_000, tags) == 10_639+++def test_headline_identical_across_harnesses_for_equivalent_work() -> None:+ """A 500-token saving reports as 500 whichever accounting shape produced it.++ Anthropic/Claude Code folds its compaction; a Codex deferral is tagged. Same+ real saving, same headline — that equivalence is the point of the helper.+ """+ anthropic_folded = headline_tokens_saved(500, {})+ codex_tagged = headline_tokens_saved(0, {"tool_search_deferred_tokens": 500})+ assert anthropic_folded == codex_tagged == 500+++def test_headline_survives_malformed_tags() -> None:+ for tags in (None, {}, "not-a-dict", {"tool_search_deferred_tokens": None}):+ assert headline_tokens_saved(10, tags) == 10+ assert headline_tokens_saved(10, {"tool_search_deferred_tokens": "abc"}) == 10+ assert headline_tokens_saved(None, None) == 0+++def test_headline_clamps_negative_message_savings() -> None:+ """Handlers revert inflation before forwarding, so a negative is a count artifact."""+ assert headline_tokens_saved(-5, {}) == 0+ assert headline_tokens_saved(-5, {"tool_search_deferred_tokens": 100}) == 95+++def test_tool_schema_compaction_saves_real_tokens_not_just_bytes() -> None:+ """The premise of folding compaction into tokens_saved on every handler.++ Compaction strips annotation keys ($schema/title/examples). If that only moved+ bytes that tokenize to nothing, the fold would be worthless — so pin a positive+ TOKEN delta on a realistically-shaped tool array, and pin that folding it into+ both endpoints keeps ``tok_before - tok_after == tok_saved`` coherent.+ """+ import json++ from headroom.providers.anthropic import AnthropicProvider+ from headroom.proxy.tool_schema_compaction import compact_tools++ tok = AnthropicProvider().get_token_counter("claude-sonnet-4-6")+ payload = {+ "tools": [+ {+ "name": f"tool_{i}",+ "description": "Does a thing.\n\n Returns text.",+ "input_schema": {+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": f"tool_{i}_schema",+ "examples": [{"path": "/tmp/x"}, {"path": "/tmp/y"}],+ "type": "object",+ "properties": {"path": {"type": "string", "description": "File path"}},+ "required": ["path"],+ },+ }+ for i in range(14)+ ]+ }+ before_tools = payload["tools"]+ body, modified, _bytes_before, _bytes_after = compact_tools(payload)+ assert modified is True++ tool_before = tok.count_text(json.dumps(before_tools, default=str))+ tool_after = tok.count_text(json.dumps(body["tools"], default=str))+ assert tool_after < tool_before, "compaction must shrink tool TOKENS, not only bytes"++ # Mirrors the fold each handler applies at its final recount, with zero message+ # compression — the shape that used to report tok_saved=0 on Claude Code.+ original_tokens = optimized_tokens = 5_000+ if 0 < tool_after < tool_before:+ original_tokens += tool_before+ optimized_tokens += tool_after+ tokens_saved = max(0, original_tokens - optimized_tokens)++ assert tokens_saved == tool_before - tool_after+ assert original_tokens - optimized_tokens == tokens_saved+ assert headline_tokens_saved(tokens_saved, {}) == tokens_saved