Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): bill the whole prompt on the gateway path so savings read true (#3632)
Tejas Choprabefdb52fd565
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.htmlindex bfe533ccd..a759fe2e5 100644--- a/headroom/dashboard/templates/dashboard.html+++ b/headroom/dashboard/templates/dashboard.html@@ -248,6 +248,9 @@<div class="mt-1 text-xs text-gray-600 leading-relaxed" x-show="(stats.tokens?.new_input_tokens || 0) > 0"><span x-text="'Of new input: ' + (stats.tokens?.new_input_savings_percent || 0).toFixed(2) + '%'" title="Savings as a fraction of the tokens that newly entered context (uncached + cache write), which is what compression can touch. The headline above recounts the session's cached history on every turn, so it reads low on a long session even when compression is working."></span></div>+ <div class="mt-1 text-xs text-gray-600 leading-relaxed">+ <span x-text="'Cost-weighted: ' + (stats.tokens?.cost_weighted_savings_percent || 0).toFixed(1) + '%'" title="The same saving in money. Lower than the token share because most removed tokens would have been cheap prefix-cache reads."></span>+ </div><div class="mt-2 h-8"><svg class="w-full h-full" viewBox="0 0 100 32" preserveAspectRatio="none" role="img":aria-label="'Savings trend sparkline, ' + (stats.savings_history || []).length + ' recent samples; latest ' + (stats.tokens?.savings_percent || 0).toFixed(1) + ' percent'">diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.pyindex 3d7e92d52..8bbf41013 100644--- a/headroom/proxy/gateway_turn.py+++ b/headroom/proxy/gateway_turn.py@@ -231,6 +231,13 @@ class NormalizedUsage:output_tokens: int | None = Nonecache_read: int | None = Nonecache_write: int | None = None+ # Whether ``input_tokens`` already counts the cached prompt. Anthropic's+ # ``input_tokens`` is the uncached remainder (reads and writes are reported+ # beside it); OpenAI's ``prompt_tokens`` is the whole prompt with+ # ``cached_tokens`` as a subset of it. Billing the prompt correctly needs to+ # know which of the two arrived, so normalize_usage records it rather than+ # letting each consumer guess.+ input_includes_cache: bool = False@propertydef has_cache_signal(self) -> bool:@@ -327,6 +334,16 @@ def normalize_usage(usage: dict[str, Any]) -> NormalizedUsage:input_tokens = _usage_int(usage, "input_tokens")if input_tokens is None:input_tokens = _usage_int(usage, "prompt_tokens")+ # Anthropic alone reports the prompt split in two: ``input_tokens`` is the+ # uncached remainder and the cached part sits in its own two counters.+ # Everywhere else the reported prompt total already contains the cached+ # prefix — including OpenAI's Responses API, which names its field+ # ``input_tokens`` too, so the key name cannot be the discriminator. Keying+ # off Anthropic's cache fields can misread only a relay that mixes the two+ # vocabularies, and then it errs towards not inflating the prompt.+ input_includes_cache = not (+ "cache_read_input_tokens" in usage or "cache_creation_input_tokens" in usage+ )output_tokens = _usage_int(usage, "output_tokens")if output_tokens is None:output_tokens = _usage_int(usage, "completion_tokens")@@ -351,6 +368,7 @@ def normalize_usage(usage: dict[str, Any]) -> NormalizedUsage:output_tokens=output_tokens,cache_read=cache_read,cache_write=cache_write,+ input_includes_cache=input_includes_cache,)@@ -1358,14 +1376,33 @@ def complete_outcome(tokenizer scale), output and the two cache counters as reported, providerstatus so a 5xx is funnelled as a failure, and the gateway's providerlatency added to the compress-side wall clock.++ ``provider_input_tokens`` is the WHOLE billed prompt, cached part included,+ matching what the direct Anthropic handler records. Anthropic's+ ``input_tokens`` counts only the uncached remainder, so on that shape the+ cache counters are added back; every other shape already contains them, so+ the cached part is subtracted to get ``uncached_input_tokens``.+ Recording only the uncached sliver here left every ratio built on the input+ counter (notably ``/stats`` ``tokens.savings_percent``) dividing savings by+ a denominator tens of times too small on cache-heavy agent traffic."""usage = usage or NormalizedUsage()+ cache_read = usage.cache_read or 0+ cache_write = usage.cache_write or 0+ reported_input = usage.input_tokens or 0+ if usage.input_includes_cache:+ billed_input = reported_input+ uncached_input = max(reported_input - cache_read - cache_write, 0)+ else:+ billed_input = reported_input + cache_read + cache_write+ uncached_input = reported_inputreturn dataclasses.replace(draft,output_tokens=usage.output_tokens or 0,- provider_input_tokens=usage.input_tokens or 0,- cache_read_tokens=usage.cache_read or 0,- cache_write_tokens=usage.cache_write or 0,+ provider_input_tokens=billed_input,+ uncached_input_tokens=uncached_input,+ cache_read_tokens=cache_read,+ cache_write_tokens=cache_write,status_code=status,total_latency_ms=draft.total_latency_ms + float(latency_ms or 0.0),)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 62d28dbd4..72104a453 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -4736,6 +4736,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# Build unified savings summary (all layers)cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)+ # Cache-aware dollar value of tool-schema deferral, the same figure the+ # cost card uses. Exposed beside the token count because tokens alone+ # mislead: deferred schemas would mostly have been cache reads, an order+ # of magnitude cheaper than list input, so a big token number is a small+ # dollar number. The token count without it invited "we removed 90% of+ # the tokens, so we saved 90% of the cost".+ _cost_tracker_stats = proxy.cost_tracker.stats() if proxy.cost_tracker else {}+ tool_schema_usd = float(_cost_tracker_stats.get("tool_savings_usd", 0.0) or 0.0)total_tokens_all_layers = all_layers_tokens_savedpersistent_savings = m.savings_tracker.stats_preview()display_session = persistent_savings.get("display_session", {})@@ -4864,13 +4872,18 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"tokens_saved": tool_schema_tokens,"requests": tool_schema_requests,"window": len(recent_request_logs),+ # Priced at what those tokens would actually have been+ # billed at (provider cache-read rate for the warm turns,+ # cache-write for the cold one), not at list input.+ "lifetime_usd": round(tool_schema_usd, 4),"description": ("Tool-definition tokens kept out of the model's context ""by deferring heavy tool schemas until they're searched ""for. Counted only when Headroom performed the deferral — ""not when the client (e.g. Claude Code / Codex) already "- "had tool search enabled. Aggregated over the recent "- "request window."+ "had tool search enabled. Token and request counts use "+ "the recent request window; lifetime_usd covers the "+ "full process lifetime."),},},@@ -4972,6 +4985,17 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:else 0,2,),+ # The same saving expressed in money rather than tokens, so a+ # reader is never left to assume the two match. They don't, and+ # not by a little: most tokens Headroom removes would have been+ # prefix-cache reads, billed at roughly a tenth of list input+ # (and less on some providers), so a large token share is a much+ # smaller cost share. Mirrors summary.cost.savings_pct, which+ # prices every layer at the rate its tokens would really have+ # cost.+ "cost_weighted_savings_percent": float(+ (summary.get("cost") or {}).get("savings_pct", 0.0) or 0.0+ ),},"latency": {"average_ms": avg_latency_ms,diff --git a/tests/gateway/test_contract_response_half.py b/tests/gateway/test_contract_response_half.pyindex 597a0265f..f0f347d73 100644--- a/tests/gateway/test_contract_response_half.py+++ b/tests/gateway/test_contract_response_half.py@@ -375,7 +375,10 @@ def test_outcome_anthropic_cache_fields(headroom_client, outcome_spy) -> None:assert o.output_tokens == 12assert o.cache_read_tokens == 2000assert o.cache_write_tokens == 500- assert o.provider_input_tokens == 300+ # Anthropic's input_tokens is the uncached remainder; the billed prompt is+ # that plus both cache buckets, as the direct handler has always recorded.+ assert o.provider_input_tokens == 2800+ assert o.uncached_input_tokens == 300def test_outcome_recorded_immediately_without_relay(headroom_client, outcome_spy) -> None:diff --git a/tests/test_gateway_turn_unit.py b/tests/test_gateway_turn_unit.pyindex 2e3f6e77b..3ccb28fb6 100644--- a/tests/test_gateway_turn_unit.py+++ b/tests/test_gateway_turn_unit.py@@ -434,7 +434,8 @@ def test_normalize_openai_chat_shape() -> None:"prompt_tokens_details": {"cached_tokens": 64},})- assert u == NormalizedUsage(100, 20, 64, None)+ # OpenAI-shaped: prompt_tokens already counts the cached prefix.+ assert u == NormalizedUsage(100, 20, 64, None, input_includes_cache=True)assert u.has_cache_signal and u.should_apply@@ -442,17 +443,19 @@ def test_normalize_openai_responses_shape() -> None:u = normalize_usage({"input_tokens": 10, "output_tokens": 2, "input_tokens_details": {"cached_tokens": 8}})- assert u == NormalizedUsage(10, 2, 8, None)+ # Responses names the field ``input_tokens`` like Anthropic but, like the+ # rest of OpenAI, already counts the cached prefix inside it.+ assert u == NormalizedUsage(10, 2, 8, None, input_includes_cache=True)def test_normalize_kong_flat_shape() -> None:u = normalize_usage({"prompt_tokens": 50, "completion_tokens": 5, "cached_tokens": 40})- assert u == NormalizedUsage(50, 5, 40, None)+ assert u == NormalizedUsage(50, 5, 40, None, input_includes_cache=True)def test_normalize_nested_usage_descends_once() -> None:u = normalize_usage({"id": "x", "usage": {"prompt_tokens": 7, "completion_tokens": 3}})- assert u == NormalizedUsage(7, 3, None, None)+ assert u == NormalizedUsage(7, 3, None, None, input_includes_cache=True)assert not u.has_cache_signal and not u.should_apply@@ -546,7 +549,10 @@ def test_complete_outcome_fills_only_existing_fields() -> None:draft = dataclasses.replace(_outcome(), total_latency_ms=5.0)done = gt.complete_outcome(draft, NormalizedUsage(120, 30, 80, 4), status=200, latency_ms=250)assert done.output_tokens == 30- assert done.provider_input_tokens == 120+ # Anthropic shape: input_tokens is the uncached remainder, so the billed+ # prompt is that plus the two cache counters.+ assert done.provider_input_tokens == 204+ assert done.uncached_input_tokens == 120assert done.optimized_tokens == 50 # never overwritten by the provider countassert (done.cache_read_tokens, done.cache_write_tokens) == (80, 4)assert done.total_latency_ms == 255.0@@ -555,6 +561,52 @@ def test_complete_outcome_fills_only_existing_fields() -> None:assert failed.status_code == 529 and failed.output_tokens == 0+def test_complete_outcome_bills_the_whole_prompt_on_a_cached_anthropic_turn() -> None:+ """A warm agent turn is ~all cache read; billing only the uncached sliver+ made ``/stats`` divide savings by a denominator tens of times too small."""+ usage = normalize_usage(+ {+ "input_tokens": 3,+ "output_tokens": 210,+ "cache_read_input_tokens": 41_000,+ "cache_creation_input_tokens": 1_200,+ }+ )+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)+ assert done.provider_input_tokens == 42_203+ assert done.uncached_input_tokens == 3+++def test_complete_outcome_does_not_double_count_openai_responses_cached_tokens() -> None:+ """The Responses shape uses Anthropic's field name for an OpenAI-style total."""+ usage = normalize_usage(+ {+ "input_tokens": 12_000,+ "output_tokens": 40,+ "input_tokens_details": {"cached_tokens": 9_000},+ }+ )+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)+ assert done.provider_input_tokens == 12_000+ assert done.uncached_input_tokens == 3_000+++def test_complete_outcome_does_not_double_count_openai_cached_tokens() -> None:+ """OpenAI's prompt_tokens already contains cached_tokens."""+ usage = normalize_usage(+ {+ "prompt_tokens": 12_000,+ "completion_tokens": 40,+ "prompt_tokens_details": {"cached_tokens": 9_000},+ }+ )+ assert usage.input_includes_cache is True+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)+ assert done.provider_input_tokens == 12_000+ assert done.uncached_input_tokens == 3_000+ assert done.cache_read_tokens == 9_000++def test_compact_gateway_tools_is_deterministic_and_isolated() -> None:tools = [{diff --git a/tests/test_stats_tool_schema_savings_are_honest.py b/tests/test_stats_tool_schema_savings_are_honest.pynew file mode 100644index 000000000..387411ef5--- /dev/null+++ b/tests/test_stats_tool_schema_savings_are_honest.py@@ -0,0 +1,86 @@+"""Tool-schema deferral must not read as a near-total saving in /stats.++Deferral withholds the same schemas on every turn, so its token count grows+with the session while the tokens it removed would, after the first turn, have+been prefix-cache reads. Two things therefore have to hold:++* the token percentage divides by every input token the provider billed, cached+ prefix included — a gateway turn that reports only its uncached remainder as+ input made this read ~97% on real Claude Code traffic;+* the same saving is published in money as well, priced at cache rates, because+ the token share and the cost share are not the same number.+"""++from __future__ import annotations++import asyncio++from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app+++def _make_client(tmp_path, monkeypatch) -> TestClient:+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))+ config = ProxyConfig(cache_enabled=False, rate_limit_enabled=False, log_requests=False)+ return TestClient(create_app(config))+++def _record_warm_agent_turns(proxy, turns: int = 20) -> None:+ """One cold turn then warm ones, each withholding the same tool schemas."""+ for turn in range(turns):+ cache_read = 0 if turn == 0 else 40_000+ cache_write = 40_000 if turn == 0 else 500+ asyncio.run(+ proxy.metrics.record_request(+ provider="anthropic",+ model="claude-sonnet-4-5",+ # The whole billed prompt, as every handler now reports it.+ input_tokens=cache_read + cache_write + 20,+ output_tokens=150,+ tokens_saved=0,+ latency_ms=10.0,+ cache_read_tokens=cache_read,+ cache_write_tokens=cache_write,+ uncached_input_tokens=20,+ tool_search_saved=10_000,+ )+ )+++def test_token_percentage_counts_the_cached_prefix_in_the_denominator(tmp_path, monkeypatch):+ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ _record_warm_agent_turns(proxy)+ tokens = client.get("/stats").json()["tokens"]++ # 200k withheld against ~800k billed: a fifth, not the ~97% that came out+ # when the denominator held only the uncached remainder of each turn.+ assert tokens["saved"] == 200_000+ assert tokens["input"] > 700_000+ assert 15.0 < tokens["savings_percent"] < 30.0+ assert tokens["savings_percent"] == tokens["all_layers_savings_percent"]+++def test_stats_publishes_the_cost_weighted_saving_next_to_the_token_saving(tmp_path, monkeypatch):+ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ _record_warm_agent_turns(proxy)+ payload = client.get("/stats").json()++ tokens = payload["tokens"]+ assert "cost_weighted_savings_percent" in tokens+ # Removed tokens are mostly cache reads, so the money share is the smaller+ # of the two. Equal values would mean the cost side priced them at list.+ assert tokens["cost_weighted_savings_percent"] < tokens["savings_percent"]+++def test_tool_schema_layer_carries_its_dollar_value(tmp_path, monkeypatch):+ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ _record_warm_agent_turns(proxy)+ layer = client.get("/stats").json()["savings"]["by_layer"]["tool_search"]++ assert layer["tokens"] >= 0 # window-scoped; the dollars are lifetime+ assert "usd" not in layer+ assert layer["lifetime_usd"] >= 0.0