Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(savings): show the new-input savings basis beside the whole-wire figure (#3485)
gglucasse92cccca00fa
diff --git a/headroom/cli/savings.py b/headroom/cli/savings.pyindex ff1bba78d..762c275c4 100644--- a/headroom/cli/savings.py+++ b/headroom/cli/savings.py@@ -47,10 +47,18 @@ def _window_line(label: str, window: dict[str, Any]) -> str:money = _money(cost)if ceiling > 0 and abs(ceiling - cost) / ceiling >= 0.01:money = f"{money} (list {_money(ceiling)})"- return (+ line = (f"{label:<12} {_bar(pct)} {pct:5.1f}% "f"saved {_tokens(saved)} / {_tokens(before)} tokens {money}")+ # Second basis, only when the window has provider cache data: the share of+ # tokens that newly entered context. The bar's ratio counts a session's+ # cached history on every turn, so a long session reads near 0% there+ # while compression is working; this is the figure the dashboard leads with.+ if int(window.get("new_input_tokens", 0) or 0) > 0:+ new_pct = float(window.get("new_input_savings_percent", 0.0) or 0.0)+ line += f" · of new input {new_pct:.1f}%"+ return line#: What each pricing basis means for a reader deciding how much to trust the@@ -125,6 +133,11 @@ def savings(as_json: bool, days: int, reset: bool) -> None:click.echo(_window_line("Today", report.windows["today"]))click.echo(_window_line("Last 7 days", report.windows["last_7_days"]))click.echo(_window_line("Last 30 days", report.windows["last_30_days"]))+ if any(int(w.get("new_input_tokens", 0) or 0) > 0 for w in report.windows.values()):+ click.echo(+ " % is of all forwarded input (cached history recounted every turn); "+ "'of new input' is of tokens that newly entered context."+ )if report.by_model:click.echo("")diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.htmlindex 3eb691532..a2aeffee5 100644--- a/headroom/dashboard/templates/dashboard.html+++ b/headroom/dashboard/templates/dashboard.html@@ -225,19 +225,21 @@<div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Tokens Saved</div><div class="flex items-baseline gap-2"><span class="text-3xl font-light tabular-nums text-accent" x-text="formatNumber(stats.tokens?.saved || 0)"></span>- <!-- Active compression ratio: savings as a fraction of tokens we- *attempted* to compress (extracted units + tool schema). When- the attempted denominator is missing (backend-routed paths- that don't yet wire per-message live-zone tracking), fall back- to the whole-request ratio so the headline doesn't collapse to- 0% while compression is happening (issue #455). -->+ <!-- Headline ratio: whole wire, savings as a fraction of ALL+ forwarded input (#1653). That is the honest answer to "what+ share of my billed input did Headroom remove", because+ without Headroom the session's cached history would have+ been billed too (as cache reads) and so belongs in the+ denominator. The new-input basis, which is what compression+ can actually touch, sits on the line below; it reads much+ higher on long sessions and must not take this slot. --><span class="text-sm text-accent" x-text="headlineSavingsPercent.toFixed(1) + '%'" :title="headlineSavingsTitle"></span></div><div class="mt-1 text-xs text-gray-500 leading-relaxed"><span x-text="'Proxy ' + formatNumber(stats.tokens?.proxy_compression_saved || 0) + ' (' + proxyShareOfTotal.toFixed(1) + '%)'"></span></div>- <div class="mt-1 text-xs text-gray-600 leading-relaxed">- <span x-text="'Of total wire: ' + (stats.tokens?.savings_percent || 0).toFixed(2) + '%'" title="Savings as fraction of all input tokens including frozen prefix"></span>+ <div class="mt-1 text-xs text-gray-600 leading-relaxed" x-show="(stats.tokens?.new_input_tokens || 0) > 0">+ <span x-text="'Of new input: ' + (stats.tokens?.new_input_savings_percent || 0).toFixed(2) + '%'" title="Savings as a fraction of the tokens that newly entered context (uncached + cache write), which is what compression can touch. The headline above recounts the session's cached history on every turn, so it reads low on a long session even when compression is working."></span></div><div class="mt-2 h-8"><svg class="w-full h-full" viewBox="0 0 100 32" preserveAspectRatio="none" role="img"@@ -2858,7 +2860,7 @@},get headlineSavingsTitle() {- return 'Of total wire input tokens';+ return 'Of total wire input tokens: all forwarded input, cached history included';},// --- Expandable Rows ---diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 3274e0e97..672b8f152 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -242,6 +242,11 @@ def build_prefix_cache_stats("cache_write_5m_requests": 0,"cache_write_1h_requests": 0,"uncached_input_tokens": 0,+ # New-input basis, from the metrics object rather than the per-provider+ # cache rows: its cohort is "newly billed input", which is not the+ # cache-activity cohort those rows are gated on.+ "new_input_tokens": int(getattr(metrics, "new_input_tokens_total", 0) or 0),+ "new_input_saved_tokens": int(getattr(metrics, "new_input_saved_tokens_total", 0) or 0),"requests": 0,"hit_requests": 0,"bust_count": 0,diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex 66cfbe7a2..2bab01ce0 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -334,6 +334,16 @@ class PrometheusMetrics:# Track per-model cache request count to distinguish cold starts from bustsself._cache_requests_by_model: dict[str, int] = defaultdict(int)+ # New-input basis. The cohort is every request that newly BILLED input+ # (uncached or cache-write tokens), which is not the same set as the+ # cache accumulators above: those are gated on cache activity, so they+ # both admit cache-read-only requests (numerator, no denominator) and+ # drop uncached-only ones (real new input, dropped entirely). The+ # ledger pairs the same two figures over the same predicate; keeping+ # one gate here is what stops /stats and `headroom savings` disagreeing.+ self.new_input_tokens_total: int = 0+ self.new_input_saved_tokens_total: int = 0+# Prefix freeze stats (cache-aware compression)self.prefix_freeze_busts_avoided: int = 0self.prefix_freeze_tokens_preserved: int = 0@@ -921,6 +931,11 @@ class PrometheusMetrics:# denominator for the active-compression ratio.self.attempted_input_tokens_total += max(0, int(attempted_input_tokens))+ # New-input cohort, on the same predicate the ledger uses below.+ if uncached_input_tokens > 0 or cache_write_tokens > 0:+ self.new_input_tokens_total += uncached_input_tokens + cache_write_tokens+ self.new_input_saved_tokens_total += max(0, int(tokens_saved))+# Track provider-specific prefix cache metricsif cache_read_tokens > 0 or cache_write_tokens > 0:pc = self.cache_by_provider[provider]@@ -1061,7 +1076,13 @@ class PrometheusMetrics:# sessions and drops deferral-only turns from the ledger entirely (#2795).deferral_saved = max(0, int(tool_search_saved))ledger_saved = tokens_saved + deferral_saved- if ledger_saved > 0 and not self._stateless:+ # A request that newly billed input is written even when it saved+ # nothing: the ledger's new-input basis needs the denominator from+ # every such request (see record_savings_event). Same predicate as the+ # `new_input_*_total` accumulators above, so the two rates share a+ # cohort — /stats and `headroom savings` are the same measurement.+ has_new_input = uncached_input_tokens > 0 or cache_write_tokens > 0+ if (ledger_saved > 0 or has_new_input) and not self._stateless:# `input_tokens` here is the optimized (post-compression) count# that was actually forwarded — see emit_request_outcome, which# passes `input_tokens=outcome.optimized_tokens`. The ledger's@@ -1104,6 +1125,15 @@ class PrometheusMetrics:uncached_input_tokens=uncached_input_tokens,cache_inferred=cache_inferred,provider=provider,+ # Provider-billed new input (the /stats new_input denominator)+ # and the deferral share of `saved`, so `headroom savings` can+ # show the same new-input rate the dashboard headline does.+ # Omitted when there is no cache breakdown (e.g. Bedrock), so+ # the ledger never divides savings by themselves.+ new_input_tokens=(+ int(uncached_input_tokens) + int(cache_write_tokens) if has_new_input else None+ ),+ deferred_tokens=deferral_saved,)otel_metrics = self._get_otel_metrics()diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 0801b5429..dbfe2247d 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -4414,9 +4414,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# any power over. Tokens Headroom removed never reached the# provider at all, so they're added back to form the baseline._pc_totals = prefix_cache_stats.get("totals", {})- new_input_tokens = int(_pc_totals.get("uncached_input_tokens", 0) or 0) + int(- _pc_totals.get("cache_write_tokens", 0) or 0- )+ new_input_tokens = int(_pc_totals.get("new_input_tokens", 0) or 0)+ # Paired numerator: savings from the SAME requests that supplied the+ # denominator, accumulated on one predicate in record_request.+ # tokens_saved_total also counts requests with no usage breakdown+ # (Bedrock, MCP tools), which would lend savings to a denominator they+ # never entered: one qualified request at 50 percent plus one+ # unqualified 10,000-token saving read as 99 percent.+ new_input_saved_tokens = int(_pc_totals.get("new_input_saved_tokens", 0) or 0)# Build human-readable summarysummary = _build_session_summary(proxy, m, prefix_cache_stats, total_tokens_before)@@ -4669,14 +4674,16 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# 200x into the denominator and long-running sessions# (1M-context models never compact) read as ~0% no# matter how well compression performs on new content.- # Guarded on new_input_tokens > 0 (not the full sum): the- # cache accumulators only see requests with cache- # activity, so a deployment with no cache metrics (e.g.- # Bedrock) would otherwise divide savings by themselves- # and report ~100%. No usage data -> report 0, not a lie.+ # Guarded on new_input_tokens > 0 (not the full sum): a+ # deployment that reports no usage breakdown at all (e.g.+ # Bedrock) contributes to neither side, and would+ # otherwise divide savings by themselves and report+ # ~100%. No usage data -> report 0, not a lie. This is+ # the same cohort `headroom savings` reports from the+ # ledger, so the two figures cannot disagree."new_input_tokens": new_input_tokens,"new_input_savings_percent": round(- (proxy_compression_tokens / (new_input_tokens + proxy_compression_tokens) * 100)+ (new_input_saved_tokens / (new_input_tokens + new_input_saved_tokens) * 100)if new_input_tokens > 0else 0,2,diff --git a/headroom/savings_ledger.py b/headroom/savings_ledger.pyindex 8e3c6bb37..44cd336a3 100644--- a/headroom/savings_ledger.py+++ b/headroom/savings_ledger.py@@ -277,6 +277,8 @@ def record_savings_event(uncached_input_tokens: Any = 0,cache_inferred: Any = False,provider: Any = None,+ new_input_tokens: int | None = None,+ deferred_tokens: int = 0,) -> bool:"""Append one savings event to the durable ledger. Never raises.@@ -287,6 +289,21 @@ def record_savings_event(proxy path does; the MCP path does not), both are recorded and the eventalso carries a cache-aware ``cost_effective_usd``. ``cost_usd`` keeps itsoriginal list-priced meaning either way, so existing readers are unaffected.+ ``new_input_tokens`` is the provider-billed input that newly entered+ context this request (uncached + cache-write), the denominator /stats+ reports as ``new_input_savings_percent``. ``deferred_tokens`` is the part+ of ``saved`` that is tool-schema deferral, which rides the cached prefix+ and is never part of new input; it is subtracted from the numerator so the+ ledger pairs compression-only savings with new input exactly as /stats+ does. Both are optional so MCP-tool events and older callers keep writing+ the same line they always did.++ A request that saved nothing but carries ``new_input_tokens`` is still+ written, as a denominator-only observation: the new-input basis must see+ every request that newly billed input, not only the ones compression+ touched, or a 100-saved/100-new turn followed by a 0-saved/10,000-new turn+ reads as 50 percent instead of under 1. Without ``new_input_tokens`` a+ zero-saving request is skipped exactly as before."""try:@@ -296,7 +313,7 @@ def record_savings_event(return Falsesaved = max(before - after, 0)- if saved <= 0:+ if saved <= 0 and new_input_tokens is None:return Falsemodel_label = _normalize_model(model)@@ -320,6 +337,13 @@ def record_savings_event("source": str(source or UNKNOWN),"pid": os.getpid(),}+ if new_input_tokens is not None:+ try:+ event["new_input"] = max(int(new_input_tokens), 0)+ event["deferred"] = min(max(int(deferred_tokens), 0), saved)+ except (TypeError, ValueError):+ event.pop("new_input", None)+ event.pop("deferred", None)# The savings split. Recorded only when the caller actually knows it —# absent means "one undifferentiated figure", which is what every v1 event@@ -443,6 +467,10 @@ class _Bucket:#: Weakest pricing basis among the events in this bucket, so a total cannot#: read as better-grounded than its worst contributing event.basis: str | None = None+ # Only events that recorded new input contribute to BOTH of these, so the+ # ratio never pairs one event's savings with another event's denominator.+ new_input_tokens: int = 0+ new_input_saved: int = 0def add(self,@@ -452,7 +480,15 @@ class _Bucket:cost: float,cost_effective: float | None = None,basis: str | None = None,+ new_input: int | None = None,+ deferred: int = 0,) -> None:+ if saved <= 0 and new_input is not None:+ # Denominator-only observation (see record_savings_event): it+ # belongs in the new-input basis and nowhere else, so calls,+ # before/saved and cost keep their saved-event semantics.+ self.new_input_tokens += new_input+ returnself.tokens_saved += savedself.tokens_before += beforeself.cost_usd += cost@@ -464,6 +500,9 @@ class _Bucket:self.cost_effective_usd += cost if cost_effective is None else cost_effectiveself.calls += 1self.basis = _weakest(self.basis, basis)+ if new_input is not None:+ self.new_input_tokens += new_input+ self.new_input_saved += max(saved - deferred, 0)@propertydef savings_percent(self) -> float:@@ -471,6 +510,20 @@ class _Bucket:return 0.0return round(self.tokens_saved / self.tokens_before * 100, 1)+ @property+ def new_input_savings_percent(self) -> float:+ """Compression-only savings as a share of what newly entered context.++ The whole-wire ratio above recounts a session's cached history every+ turn, so long sessions read as ~0% no matter how well compression does+ on new content. Same definition as /stats ``new_input_savings_percent``:+ saved / (new input + saved), since removed tokens never reached the+ provider and are added back to form the baseline.+ """+ if self.new_input_tokens <= 0:+ return 0.0+ return round(self.new_input_saved / (self.new_input_tokens + self.new_input_saved) * 100, 1)+def to_dict(self) -> dict[str, Any]:return {"tokens_saved": self.tokens_saved,@@ -480,6 +533,8 @@ class _Bucket:"basis": self.basis or BASIS_LIST,"calls": self.calls,"savings_percent": self.savings_percent,+ "new_input_tokens": self.new_input_tokens,+ "new_input_savings_percent": self.new_input_savings_percent,}@@ -590,6 +645,15 @@ def aggregate_savings(# built from them widens to a union that no longer type-checks against# `add`'s signature. Listed out per bucket rather than via a local# helper, which would close over these loop variables.+ new_input: int | None = None+ deferred = 0+ if event.get("new_input") is not None:+ try:+ new_input = max(int(event.get("new_input") or 0), 0)+ deferred = max(int(event.get("deferred", 0) or 0), 0)+ except (TypeError, ValueError):+ new_input = None+targets = [windowed]if ts >= today_cutoff:targets.append(today)@@ -605,6 +669,8 @@ def aggregate_savings(cost=cost,cost_effective=effective,basis=basis,+ new_input=new_input,+ deferred=deferred,)model_rows = _ranked(by_model, "model")diff --git a/tests/test_dashboard_token_savings.py b/tests/test_dashboard_token_savings.pyindex 6c8b02569..c2aa68794 100644--- a/tests/test_dashboard_token_savings.py+++ b/tests/test_dashboard_token_savings.py@@ -13,15 +13,33 @@ def _getter_body(html: str, name: str) -> str:return match.group("body")-def test_token_savings_headline_uses_total_wire_denominator() -> None:+def test_token_savings_headline_leads_with_total_wire_then_new_input() -> None:html = get_dashboard_html()+ # Whole wire keeps the headline slot (#1653): it is the conservative,+ # screenshot-able figure. The old attempted-denominator must stay gone.headline_body = _getter_body(html, "headlineSavingsPercent")assert "stats.tokens?.savings_percent" in headline_bodyassert "stats.tokens?.proxy_savings_percent" in headline_body+ assert "stats.tokens?.new_input_savings_percent" not in headline_bodyassert "stats.tokens?.active_savings_percent" not in headline_bodyassert "stats.tokens?.proxy_attempted_tokens" not in headline_bodytitle_body = _getter_body(html, "headlineSavingsTitle")assert "Of total wire input tokens" in title_bodyassert "Of compressible tokens attempted" not in title_body++ # New input rides the line below, gated on a provider cache breakdown, and+ # replaces the old "Of total wire" sublabel that duplicated the headline.+ assert "'Of new input: '" in html+ assert "'Of total wire: '" not in html+ assert 'x-show="(stats.tokens?.new_input_tokens || 0) > 0"' in html+++def test_new_input_explanation_lives_in_the_tooltip_not_the_card() -> None:+ """The card shows a number and its basis; the "why" is hover-only."""+ html = get_dashboard_html()+ card_line = next(line for line in html.splitlines() if "'Of new input: '" in line)+ label, _, tooltip = card_line.partition("title=")+ assert "what compression can touch" not in label+ assert "what compression can touch" in tooltipdiff --git a/tests/test_savings_ledger.py b/tests/test_savings_ledger.pyindex e51c7ef33..41b5b1e73 100644--- a/tests/test_savings_ledger.py+++ b/tests/test_savings_ledger.py@@ -115,6 +115,50 @@ def test_windows_today_week_last30(monkeypatch, tmp_path):assert report.windows["today"]["savings_percent"] == pytest.approx(50.0)+def test_new_input_basis_pairs_compression_only_with_new_input(monkeypatch, tmp_path):+ _events_env(monkeypatch, tmp_path)+ # 400 saved, of which 100 is tool-schema deferral; 900 tokens newly entered+ # context. Compression-only 300 / (900 + 300) = 25%, same as /stats.+ L.record_savings_event(+ tokens_before=10_000,+ tokens_after=9_600,+ model=None,+ client="claude-code",+ new_input_tokens=900,+ deferred_tokens=100,+ )+ # An event without a cache breakdown (MCP tool, Bedrock) keeps writing the+ # old line and must not lend its savings to the new-input ratio.+ L.record_savings_event(tokens_before=1000, tokens_after=500, model=None, client="mcp")+ report = L.aggregate_savings()+ window = report.windows["today"]+ assert window["tokens_saved"] == 900+ assert window["new_input_tokens"] == 900+ assert window["new_input_savings_percent"] == pytest.approx(25.0)+ clients = {row["client"]: row for row in report.by_client}+ assert clients["mcp"]["new_input_tokens"] == 0+ assert clients["mcp"]["new_input_savings_percent"] == 0.0+ # The whole-wire ratio is untouched by the new field.+ assert window["savings_percent"] == pytest.approx(900 / 11_000 * 100, abs=0.1)+++def test_savings_cli_prints_new_input_line_only_with_cache_data(monkeypatch, tmp_path):+ from click.testing import CliRunner++ from headroom.cli.savings import savings++ _events_env(monkeypatch, tmp_path)+ L.record_savings_event(tokens_before=1000, tokens_after=500, model=None, client="mcp")+ out = CliRunner().invoke(savings, []).output+ assert "of new input" not in out+ L.record_savings_event(+ tokens_before=10_000, tokens_after=9_600, model=None, client="c", new_input_tokens=900+ )+ out = CliRunner().invoke(savings, []).output+ assert "of new input 30.8%" in out+ assert "newly entered context" in out++def test_retention_hard_capped_at_30_days(monkeypatch, tmp_path):_events_env(monkeypatch, tmp_path)now = datetime(2026, 6, 17, 12, 0, tzinfo=UTC)@@ -450,3 +494,30 @@ def test_v1_events_still_aggregate_alongside_v2(tmp_path):# ...but the v1 event's list dollar is still counted, not dropped.assert lifetime["cost_effective_usd"] > 0.012assert lifetime["basis"] == "list"+++def test_zero_saving_request_keeps_its_place_in_the_new_input_denominator(monkeypatch, tmp_path):+ """100 saved on 100 new input, then 0 saved on 10,000 new input. The+ new-input basis is 100 / (10,100 + 100), under 1 percent, not the 50+ percent that only counting requests that saved would report. The+ saved-event figures do not see the second request at all."""+ _events_env(monkeypatch, tmp_path)+ assert L.record_savings_event(+ tokens_before=200, tokens_after=100, model=None, client="claude-code", new_input_tokens=100+ )+ assert L.record_savings_event(+ tokens_before=10_000,+ tokens_after=10_000,+ model=None,+ client="claude-code",+ new_input_tokens=10_000,+ )+ # Without a new-input figure a zero-saving request is still not an event.+ assert not L.record_savings_event(+ tokens_before=500, tokens_after=500, model=None, client="claude-code"+ )+ window = L.aggregate_savings().windows["today"]+ assert window["tokens_saved"] == 100+ assert window["new_input_tokens"] == 10_100+ assert window["new_input_savings_percent"] == pytest.approx(1.0, abs=0.05)+ assert window["savings_percent"] == pytest.approx(50.0)diff --git a/tests/test_savings_ledger_before_forwarded.py b/tests/test_savings_ledger_before_forwarded.pyindex 7e61940e4..eed460543 100644--- a/tests/test_savings_ledger_before_forwarded.py+++ b/tests/test_savings_ledger_before_forwarded.py@@ -73,6 +73,27 @@ async def test_record_savings_event_uses_original_input_as_before(assert call["saved_compression"] == 400assert call["saved_tool_schema"] == 0+ # With a provider cache breakdown the ledger also gets the /stats+ # new-input denominator (uncached + cache write) and the deferral share of+ # the saving, so `headroom savings` can pair compression-only with new input.+ calls.clear()+ await metrics.record_request(+ provider="anthropic",+ model="claude-opus-4-6",+ input_tokens=600,+ output_tokens=25,+ tokens_saved=400,+ latency_ms=10.0,+ client="claude-code",+ cache_read_tokens=5000,+ cache_write_tokens=700,+ uncached_input_tokens=200,+ tool_search_saved=150,+ )+ assert calls[0]["tokens_before"] == 1000 + 150+ assert calls[0]["new_input_tokens"] == 900+ assert calls[0]["deferred_tokens"] == 150+@pytest.mark.asyncioasync def test_record_request_forwards_tool_savings_to_otel() -> None:@@ -161,3 +182,46 @@ async def test_record_savings_event_written_for_deferral_only_turn(assert len(calls) == 1assert calls[0]["tokens_before"] == 50000 + 13182assert calls[0]["tokens_after"] == 50000+++async def test_zero_saving_request_with_cache_breakdown_reaches_the_ledger(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A request that saved nothing but newly billed input is a denominator+ observation for the new-input basis, so it must reach the ledger. One+ without any breakdown is skipped as before."""+ calls: list[dict[str, Any]] = []+ monkeypatch.setattr(+ prometheus_metrics.savings_ledger,+ "record_savings_event",+ lambda **kwargs: calls.append(kwargs),+ )+ metrics = prometheus_metrics.PrometheusMetrics(+ savings_tracker=_FakeSavingsTracker(),+ otel_metrics=_FakeOtelMetrics(),+ )+ await metrics.record_request(+ provider="anthropic",+ model="claude-opus-4-6",+ input_tokens=10_000,+ output_tokens=25,+ tokens_saved=0,+ latency_ms=10.0,+ client="claude-code",+ cache_read_tokens=5_000,+ uncached_input_tokens=10_000,+ )+ assert len(calls) == 1+ assert calls[0]["tokens_before"] == calls[0]["tokens_after"]+ assert calls[0]["new_input_tokens"] == 10_000+ await metrics.record_request(+ provider="bedrock",+ model="claude-opus-4-6",+ input_tokens=10_000,+ output_tokens=25,+ tokens_saved=0,+ latency_ms=10.0,+ client="claude-code",+ )+ assert len(calls) == 1diff --git a/tests/test_stats_new_input_savings_rate.py b/tests/test_stats_new_input_savings_rate.pyindex f566a6d4a..14b94881b 100644--- a/tests/test_stats_new_input_savings_rate.py+++ b/tests/test_stats_new_input_savings_rate.py@@ -11,8 +11,10 @@ from __future__ import annotationsimport asyncio+import pytestfrom fastapi.testclient import TestClient+from headroom import savings_ledgerfrom headroom.proxy.server import ProxyConfig, create_app@@ -79,3 +81,98 @@ def test_stats_new_input_rate_is_zero_without_cache_usage_data(tmp_path, monkeypassert tokens["new_input_tokens"] == 0assert tokens["new_input_savings_percent"] == 0+++def test_stats_new_input_rate_pairs_savings_with_qualified_requests(tmp_path, monkeypatch):+ """The numerator must come from the same requests as the denominator. A+ request with no cache breakdown (Bedrock, an MCP tool) never enters+ new_input_tokens, so its savings must not lend themselves to that ratio:+ one qualified 50 percent request plus one unqualified 10,000-token saving+ used to read as 99 percent."""+ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ asyncio.run(+ proxy.metrics.record_request(+ provider="anthropic",+ model="claude-opus-4-6",+ input_tokens=1_100,+ output_tokens=10,+ tokens_saved=100,+ latency_ms=10.0,+ cache_read_tokens=1_000,+ uncached_input_tokens=100,+ )+ )+ assert client.get("/stats").json()["tokens"]["new_input_savings_percent"] == 50.0+ asyncio.run(+ proxy.metrics.record_request(+ provider="bedrock",+ model="claude-opus-4-6",+ input_tokens=20_000,+ output_tokens=10,+ tokens_saved=10_000,+ latency_ms=10.0,+ )+ )+ tokens = client.get("/stats").json()["tokens"]+ assert tokens["new_input_tokens"] == 100+ assert tokens["new_input_savings_percent"] == 50.0+ # The whole-wire figures still count the unqualified request.+ assert tokens["saved"] >= 10_100+++def test_stats_and_ledger_share_one_new_input_cohort(tmp_path, monkeypatch):+ """The dashboard rate and `headroom savings` must be the same measurement.++ /stats accumulated its new-input pair under a cache-ACTIVITY gate while the+ ledger writes under a newly-BILLED-input one, so the two admitted different+ requests. An uncached-only turn (real new input, no cache read or write)+ reached the ledger and nothing else, and a cache-read-only turn lent its+ savings to the dashboard numerator with no denominator to match. The+ reviewer's case: 100 saved / 100 new / 1,000 cache-read followed by+ 0 saved / 10,000 new left the dashboard at 50% and the ledger near 1%.+ """+ ledger_path = tmp_path / "savings_events.jsonl"+ monkeypatch.setattr(savings_ledger, "_resolve_path", lambda path=None: ledger_path)++ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy++ async def record(**kwargs) -> None:+ await proxy.metrics.record_request(+ provider="anthropic",+ model="claude-opus-4-6",+ output_tokens=10,+ latency_ms=10.0,+ **kwargs,+ )++ # Cache-read plus a little new input: in both cohorts.+ asyncio.run(+ record(+ input_tokens=1_100,+ tokens_saved=100,+ cache_read_tokens=1_000,+ uncached_input_tokens=100,+ )+ )+ # Uncached-only: real new input, no cache activity at all. Used to be+ # a ledger-only observation.+ asyncio.run(record(input_tokens=10_000, tokens_saved=0, uncached_input_tokens=10_000))+ # Cache-read-only: no new input, so it belongs to neither side despite+ # saving tokens. Used to inflate the dashboard numerator alone.+ asyncio.run(record(input_tokens=2_000, tokens_saved=500, cache_read_tokens=2_000))++ tokens = client.get("/stats").json()["tokens"]++ lifetime = savings_ledger.aggregate_savings(path=ledger_path).lifetime++ assert tokens["new_input_tokens"] == 10_100+ assert lifetime["new_input_tokens"] == 10_100+ # 100 saved / (10,100 new + 100 saved) — not the old 50%.+ assert tokens["new_input_savings_percent"] == pytest.approx(0.98, abs=0.05)+ assert lifetime["new_input_savings_percent"] == pytest.approx(+ tokens["new_input_savings_percent"], abs=0.05+ )+ # The cache-read-only turn's savings still count on the whole-wire figure.+ assert tokens["saved"] == 600