Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): bill the whole prompt on the gateway path so savings read true (#3632)

Tejas Choprabefdb52fd565
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.html
index bfe533ccd..a759fe2e5 100644
--- a/headroom/dashboard/templates/dashboard.html
+++ b/headroom/dashboard/templates/dashboard.html
@@ -248,6 +248,9 @@
<div class="mt-1 text-xs text-gray-600 leading-relaxed" x-show="(stats.tokens?.new_input_tokens || 0) > 0">
<span x-text="'Of new input: ' + (stats.tokens?.new_input_savings_percent || 0).toFixed(2) + '%'" title="Savings as a fraction of the tokens that newly entered context (uncached + cache write), which is what compression can touch. The headline above recounts the session's cached history on every turn, so it reads low on a long session even when compression is working."></span>
</div>
+ <div class="mt-1 text-xs text-gray-600 leading-relaxed">
+ <span x-text="'Cost-weighted: ' + (stats.tokens?.cost_weighted_savings_percent || 0).toFixed(1) + '%'" title="The same saving in money. Lower than the token share because most removed tokens would have been cheap prefix-cache reads."></span>
+ </div>
<div class="mt-2 h-8">
<svg class="w-full h-full" viewBox="0 0 100 32" preserveAspectRatio="none" role="img"
:aria-label="'Savings trend sparkline, ' + (stats.savings_history || []).length + ' recent samples; latest ' + (stats.tokens?.savings_percent || 0).toFixed(1) + ' percent'">
diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.py
index 3d7e92d52..8bbf41013 100644
--- a/headroom/proxy/gateway_turn.py
+++ b/headroom/proxy/gateway_turn.py
@@ -231,6 +231,13 @@ class NormalizedUsage:
output_tokens: int | None = None
cache_read: int | None = None
cache_write: int | None = None
+ # Whether ``input_tokens`` already counts the cached prompt. Anthropic's
+ # ``input_tokens`` is the uncached remainder (reads and writes are reported
+ # beside it); OpenAI's ``prompt_tokens`` is the whole prompt with
+ # ``cached_tokens`` as a subset of it. Billing the prompt correctly needs to
+ # know which of the two arrived, so normalize_usage records it rather than
+ # letting each consumer guess.
+ input_includes_cache: bool = False
@property
def has_cache_signal(self) -> bool:
@@ -327,6 +334,16 @@ def normalize_usage(usage: dict[str, Any]) -> NormalizedUsage:
input_tokens = _usage_int(usage, "input_tokens")
if input_tokens is None:
input_tokens = _usage_int(usage, "prompt_tokens")
+ # Anthropic alone reports the prompt split in two: ``input_tokens`` is the
+ # uncached remainder and the cached part sits in its own two counters.
+ # Everywhere else the reported prompt total already contains the cached
+ # prefix — including OpenAI's Responses API, which names its field
+ # ``input_tokens`` too, so the key name cannot be the discriminator. Keying
+ # off Anthropic's cache fields can misread only a relay that mixes the two
+ # vocabularies, and then it errs towards not inflating the prompt.
+ input_includes_cache = not (
+ "cache_read_input_tokens" in usage or "cache_creation_input_tokens" in usage
+ )
output_tokens = _usage_int(usage, "output_tokens")
if output_tokens is None:
output_tokens = _usage_int(usage, "completion_tokens")
@@ -351,6 +368,7 @@ def normalize_usage(usage: dict[str, Any]) -> NormalizedUsage:
output_tokens=output_tokens,
cache_read=cache_read,
cache_write=cache_write,
+ input_includes_cache=input_includes_cache,
)
@@ -1358,14 +1376,33 @@ def complete_outcome(
tokenizer scale), output and the two cache counters as reported, provider
status so a 5xx is funnelled as a failure, and the gateway's provider
latency added to the compress-side wall clock.
+
+ ``provider_input_tokens`` is the WHOLE billed prompt, cached part included,
+ matching what the direct Anthropic handler records. Anthropic's
+ ``input_tokens`` counts only the uncached remainder, so on that shape the
+ cache counters are added back; every other shape already contains them, so
+ the cached part is subtracted to get ``uncached_input_tokens``.
+ Recording only the uncached sliver here left every ratio built on the input
+ counter (notably ``/stats`` ``tokens.savings_percent``) dividing savings by
+ a denominator tens of times too small on cache-heavy agent traffic.
"""
usage = usage or NormalizedUsage()
+ cache_read = usage.cache_read or 0
+ cache_write = usage.cache_write or 0
+ reported_input = usage.input_tokens or 0
+ if usage.input_includes_cache:
+ billed_input = reported_input
+ uncached_input = max(reported_input - cache_read - cache_write, 0)
+ else:
+ billed_input = reported_input + cache_read + cache_write
+ uncached_input = reported_input
return dataclasses.replace(
draft,
output_tokens=usage.output_tokens or 0,
- provider_input_tokens=usage.input_tokens or 0,
- cache_read_tokens=usage.cache_read or 0,
- cache_write_tokens=usage.cache_write or 0,
+ provider_input_tokens=billed_input,
+ uncached_input_tokens=uncached_input,
+ cache_read_tokens=cache_read,
+ cache_write_tokens=cache_write,
status_code=status,
total_latency_ms=draft.total_latency_ms + float(latency_ms or 0.0),
)
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 62d28dbd4..72104a453 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -4736,6 +4736,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# Build unified savings summary (all layers)
cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
+ # Cache-aware dollar value of tool-schema deferral, the same figure the
+ # cost card uses. Exposed beside the token count because tokens alone
+ # mislead: deferred schemas would mostly have been cache reads, an order
+ # of magnitude cheaper than list input, so a big token number is a small
+ # dollar number. The token count without it invited "we removed 90% of
+ # the tokens, so we saved 90% of the cost".
+ _cost_tracker_stats = proxy.cost_tracker.stats() if proxy.cost_tracker else {}
+ tool_schema_usd = float(_cost_tracker_stats.get("tool_savings_usd", 0.0) or 0.0)
total_tokens_all_layers = all_layers_tokens_saved
persistent_savings = m.savings_tracker.stats_preview()
display_session = persistent_savings.get("display_session", {})
@@ -4864,13 +4872,18 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"tokens_saved": tool_schema_tokens,
"requests": tool_schema_requests,
"window": len(recent_request_logs),
+ # Priced at what those tokens would actually have been
+ # billed at (provider cache-read rate for the warm turns,
+ # cache-write for the cold one), not at list input.
+ "lifetime_usd": round(tool_schema_usd, 4),
"description": (
"Tool-definition tokens kept out of the model's context "
"by deferring heavy tool schemas until they're searched "
"for. Counted only when Headroom performed the deferral — "
"not when the client (e.g. Claude Code / Codex) already "
- "had tool search enabled. Aggregated over the recent "
- "request window."
+ "had tool search enabled. Token and request counts use "
+ "the recent request window; lifetime_usd covers the "
+ "full process lifetime."
),
},
},
@@ -4972,6 +4985,17 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
else 0,
2,
),
+ # The same saving expressed in money rather than tokens, so a
+ # reader is never left to assume the two match. They don't, and
+ # not by a little: most tokens Headroom removes would have been
+ # prefix-cache reads, billed at roughly a tenth of list input
+ # (and less on some providers), so a large token share is a much
+ # smaller cost share. Mirrors summary.cost.savings_pct, which
+ # prices every layer at the rate its tokens would really have
+ # cost.
+ "cost_weighted_savings_percent": float(
+ (summary.get("cost") or {}).get("savings_pct", 0.0) or 0.0
+ ),
},
"latency": {
"average_ms": avg_latency_ms,
diff --git a/tests/gateway/test_contract_response_half.py b/tests/gateway/test_contract_response_half.py
index 597a0265f..f0f347d73 100644
--- a/tests/gateway/test_contract_response_half.py
+++ b/tests/gateway/test_contract_response_half.py
@@ -375,7 +375,10 @@ def test_outcome_anthropic_cache_fields(headroom_client, outcome_spy) -> None:
assert o.output_tokens == 12
assert o.cache_read_tokens == 2000
assert o.cache_write_tokens == 500
- assert o.provider_input_tokens == 300
+ # Anthropic's input_tokens is the uncached remainder; the billed prompt is
+ # that plus both cache buckets, as the direct handler has always recorded.
+ assert o.provider_input_tokens == 2800
+ assert o.uncached_input_tokens == 300
def test_outcome_recorded_immediately_without_relay(headroom_client, outcome_spy) -> None:
diff --git a/tests/test_gateway_turn_unit.py b/tests/test_gateway_turn_unit.py
index 2e3f6e77b..3ccb28fb6 100644
--- a/tests/test_gateway_turn_unit.py
+++ b/tests/test_gateway_turn_unit.py
@@ -434,7 +434,8 @@ def test_normalize_openai_chat_shape() -> None:
"prompt_tokens_details": {"cached_tokens": 64},
}
)
- assert u == NormalizedUsage(100, 20, 64, None)
+ # OpenAI-shaped: prompt_tokens already counts the cached prefix.
+ assert u == NormalizedUsage(100, 20, 64, None, input_includes_cache=True)
assert u.has_cache_signal and u.should_apply
@@ -442,17 +443,19 @@ def test_normalize_openai_responses_shape() -> None:
u = normalize_usage(
{"input_tokens": 10, "output_tokens": 2, "input_tokens_details": {"cached_tokens": 8}}
)
- assert u == NormalizedUsage(10, 2, 8, None)
+ # Responses names the field ``input_tokens`` like Anthropic but, like the
+ # rest of OpenAI, already counts the cached prefix inside it.
+ assert u == NormalizedUsage(10, 2, 8, None, input_includes_cache=True)
def test_normalize_kong_flat_shape() -> None:
u = normalize_usage({"prompt_tokens": 50, "completion_tokens": 5, "cached_tokens": 40})
- assert u == NormalizedUsage(50, 5, 40, None)
+ assert u == NormalizedUsage(50, 5, 40, None, input_includes_cache=True)
def test_normalize_nested_usage_descends_once() -> None:
u = normalize_usage({"id": "x", "usage": {"prompt_tokens": 7, "completion_tokens": 3}})
- assert u == NormalizedUsage(7, 3, None, None)
+ assert u == NormalizedUsage(7, 3, None, None, input_includes_cache=True)
assert not u.has_cache_signal and not u.should_apply
@@ -546,7 +549,10 @@ def test_complete_outcome_fills_only_existing_fields() -> None:
draft = dataclasses.replace(_outcome(), total_latency_ms=5.0)
done = gt.complete_outcome(draft, NormalizedUsage(120, 30, 80, 4), status=200, latency_ms=250)
assert done.output_tokens == 30
- assert done.provider_input_tokens == 120
+ # Anthropic shape: input_tokens is the uncached remainder, so the billed
+ # prompt is that plus the two cache counters.
+ assert done.provider_input_tokens == 204
+ assert done.uncached_input_tokens == 120
assert done.optimized_tokens == 50 # never overwritten by the provider count
assert (done.cache_read_tokens, done.cache_write_tokens) == (80, 4)
assert done.total_latency_ms == 255.0
@@ -555,6 +561,52 @@ def test_complete_outcome_fills_only_existing_fields() -> None:
assert failed.status_code == 529 and failed.output_tokens == 0
+def test_complete_outcome_bills_the_whole_prompt_on_a_cached_anthropic_turn() -> None:
+ """A warm agent turn is ~all cache read; billing only the uncached sliver
+ made ``/stats`` divide savings by a denominator tens of times too small."""
+ usage = normalize_usage(
+ {
+ "input_tokens": 3,
+ "output_tokens": 210,
+ "cache_read_input_tokens": 41_000,
+ "cache_creation_input_tokens": 1_200,
+ }
+ )
+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)
+ assert done.provider_input_tokens == 42_203
+ assert done.uncached_input_tokens == 3
+
+
+def test_complete_outcome_does_not_double_count_openai_responses_cached_tokens() -> None:
+ """The Responses shape uses Anthropic's field name for an OpenAI-style total."""
+ usage = normalize_usage(
+ {
+ "input_tokens": 12_000,
+ "output_tokens": 40,
+ "input_tokens_details": {"cached_tokens": 9_000},
+ }
+ )
+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)
+ assert done.provider_input_tokens == 12_000
+ assert done.uncached_input_tokens == 3_000
+
+
+def test_complete_outcome_does_not_double_count_openai_cached_tokens() -> None:
+ """OpenAI's prompt_tokens already contains cached_tokens."""
+ usage = normalize_usage(
+ {
+ "prompt_tokens": 12_000,
+ "completion_tokens": 40,
+ "prompt_tokens_details": {"cached_tokens": 9_000},
+ }
+ )
+ assert usage.input_includes_cache is True
+ done = gt.complete_outcome(_outcome(), usage, status=200, latency_ms=100)
+ assert done.provider_input_tokens == 12_000
+ assert done.uncached_input_tokens == 3_000
+ assert done.cache_read_tokens == 9_000
+
+
def test_compact_gateway_tools_is_deterministic_and_isolated() -> None:
tools = [
{
diff --git a/tests/test_stats_tool_schema_savings_are_honest.py b/tests/test_stats_tool_schema_savings_are_honest.py
new file mode 100644
index 000000000..387411ef5
--- /dev/null
+++ b/tests/test_stats_tool_schema_savings_are_honest.py
@@ -0,0 +1,86 @@
+"""Tool-schema deferral must not read as a near-total saving in /stats.
+
+Deferral withholds the same schemas on every turn, so its token count grows
+with the session while the tokens it removed would, after the first turn, have
+been prefix-cache reads. Two things therefore have to hold:
+
+* the token percentage divides by every input token the provider billed, cached
+ prefix included — a gateway turn that reports only its uncached remainder as
+ input made this read ~97% on real Claude Code traffic;
+* the same saving is published in money as well, priced at cache rates, because
+ the token share and the cost share are not the same number.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from fastapi.testclient import TestClient
+
+from headroom.proxy.server import ProxyConfig, create_app
+
+
+def _make_client(tmp_path, monkeypatch) -> TestClient:
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))
+ config = ProxyConfig(cache_enabled=False, rate_limit_enabled=False, log_requests=False)
+ return TestClient(create_app(config))
+
+
+def _record_warm_agent_turns(proxy, turns: int = 20) -> None:
+ """One cold turn then warm ones, each withholding the same tool schemas."""
+ for turn in range(turns):
+ cache_read = 0 if turn == 0 else 40_000
+ cache_write = 40_000 if turn == 0 else 500
+ asyncio.run(
+ proxy.metrics.record_request(
+ provider="anthropic",
+ model="claude-sonnet-4-5",
+ # The whole billed prompt, as every handler now reports it.
+ input_tokens=cache_read + cache_write + 20,
+ output_tokens=150,
+ tokens_saved=0,
+ latency_ms=10.0,
+ cache_read_tokens=cache_read,
+ cache_write_tokens=cache_write,
+ uncached_input_tokens=20,
+ tool_search_saved=10_000,
+ )
+ )
+
+
+def test_token_percentage_counts_the_cached_prefix_in_the_denominator(tmp_path, monkeypatch):
+ with _make_client(tmp_path, monkeypatch) as client:
+ proxy = client.app.state.proxy
+ _record_warm_agent_turns(proxy)
+ tokens = client.get("/stats").json()["tokens"]
+
+ # 200k withheld against ~800k billed: a fifth, not the ~97% that came out
+ # when the denominator held only the uncached remainder of each turn.
+ assert tokens["saved"] == 200_000
+ assert tokens["input"] > 700_000
+ assert 15.0 < tokens["savings_percent"] < 30.0
+ assert tokens["savings_percent"] == tokens["all_layers_savings_percent"]
+
+
+def test_stats_publishes_the_cost_weighted_saving_next_to_the_token_saving(tmp_path, monkeypatch):
+ with _make_client(tmp_path, monkeypatch) as client:
+ proxy = client.app.state.proxy
+ _record_warm_agent_turns(proxy)
+ payload = client.get("/stats").json()
+
+ tokens = payload["tokens"]
+ assert "cost_weighted_savings_percent" in tokens
+ # Removed tokens are mostly cache reads, so the money share is the smaller
+ # of the two. Equal values would mean the cost side priced them at list.
+ assert tokens["cost_weighted_savings_percent"] < tokens["savings_percent"]
+
+
+def test_tool_schema_layer_carries_its_dollar_value(tmp_path, monkeypatch):
+ with _make_client(tmp_path, monkeypatch) as client:
+ proxy = client.app.state.proxy
+ _record_warm_agent_turns(proxy)
+ layer = client.get("/stats").json()["savings"]["by_layer"]["tool_search"]
+
+ assert layer["tokens"] >= 0 # window-scoped; the dollars are lifetime
+ assert "usd" not in layer
+ assert layer["lifetime_usd"] >= 0.0