Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(proxy): full-savings cost card, cache-aware tool-schema pricing, routing-stats seam (#3351)

Tejas Choprab8fa3ad9bdf7
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.html
index 53e943bc8..81f8c8cb4 100644
--- a/headroom/dashboard/templates/dashboard.html
+++ b/headroom/dashboard/templates/dashboard.html
@@ -864,6 +864,56 @@
</template>
</div>
+ <!-- Model Routing: shown only when a routing-stats provider is
+ registered (stats.routing; see headroom.proxy.routing_stats). -->
+ <template x-if="(stats.routing?.pairs || []).length > 0">
+ <div class="bg-surface rounded-lg p-4 border border-border mb-6">
+ <div class="flex items-center justify-between mb-3">
+ <span class="text-sm font-medium text-gray-300">Model Routing</span>
+ <span class="text-xs text-gray-500 font-mono"
+ x-text="(stats.routing.downgrades || 0) + ' downgraded · ' + (stats.routing.upgrades || 0) + ' upgraded · ' + (stats.routing.unchanged || 0) + ' unchanged · ' + formatNumber(stats.routing.decisions || 0) + ' decisions'"></span>
+ </div>
+ <div class="overflow-x-auto">
+ <table class="w-full text-sm">
+ <thead>
+ <tr class="text-xs text-gray-500 uppercase tracking-wide text-left">
+ <th class="py-1 pr-2 font-medium">Requested</th>
+ <th class="py-1 pr-2 font-medium"></th>
+ <th class="py-1 pr-4 font-medium">Served</th>
+ <th class="py-1 pr-4 font-medium text-right">Requests</th>
+ <th class="py-1 pr-4 font-medium text-right">Enforced</th>
+ <th class="py-1 pr-4 font-medium text-right">Holdout</th>
+ <th class="py-1 pr-4 font-medium text-right">Measured</th>
+ <th class="py-1 font-medium text-right">Saved</th>
+ </tr>
+ </thead>
+ <tbody>
+ <template x-for="p in stats.routing.pairs" :key="p.requested + '>' + p.served">
+ <tr class="border-t border-border">
+ <td class="py-1.5 pr-2 font-mono text-xs" x-text="p.requested"></td>
+ <td class="py-1.5 pr-2 text-xs"
+ :class="p.direction === 'downgrade' ? 'text-emerald-400' : (p.direction === 'upgrade' ? 'text-amber-400' : 'text-gray-500')"
+ x-text="p.direction === 'downgrade' ? '↓' : (p.direction === 'upgrade' ? '↑' : '→')"></td>
+ <td class="py-1.5 pr-4 font-mono text-xs" x-text="p.served"></td>
+ <td class="py-1.5 pr-4 text-right tabular-nums" x-text="formatNumber(p.count)"></td>
+ <td class="py-1.5 pr-4 text-right tabular-nums" x-text="formatNumber(p.enforced)"></td>
+ <td class="py-1.5 pr-4 text-right tabular-nums text-gray-500" x-text="formatNumber(p.holdout)"></td>
+ <td class="py-1.5 pr-4 text-right tabular-nums text-gray-500" x-text="formatNumber(p.measured)"></td>
+ <td class="py-1.5 text-right tabular-nums"
+ :class="p.savings_usd > 0 ? 'text-emerald-400' : (p.savings_usd < 0 ? 'text-red-400' : 'text-gray-500')"
+ x-text="(p.savings_usd < 0 ? '-$' : '$') + formatCurrency(Math.abs(p.savings_usd || 0))"></td>
+ </tr>
+ </template>
+ </tbody>
+ </table>
+ </div>
+ <div class="mt-2 text-xs text-gray-600">
+ Saved = measured baseline-model cost minus served cost, on rerouted rows with provider-reported usage.
+ The requested model is a ceiling; holdout rows stay unrouted as the control arm.
+ </div>
+ </div>
+ </template>
+
<!-- Recent Requests Table (with expandable rows) -->
<div class="bg-surface rounded-lg border border-border overflow-hidden mb-6">
<div class="px-4 py-3 border-b border-border flex justify-between items-center">
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 5f7dd839e..efe5a7169 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -590,14 +590,34 @@ def build_session_summary(
best_compression = best["savings_pct"]
best_detail = f"{best['original']:,} → {best['optimized']:,} tokens"
- # Cost summary — dollar savings are proxy-compression only at model list
- # price.
+ # Cost summary — every savings layer priced into one card. Measured layers
+ # (compression, tool-schema deferral, prefix-cache net) and
+ # extension-attributed dollars are itemized in ``breakdown``; rows the
+ # attribution ledger flags ``realized=False`` (e.g. a model-routing
+ # extension's decision-time delta) are summed under a ``projected`` key so
+ # a reader can still split measured from estimated. Budget enforcement
+ # keeps reading the unwidened ``savings_usd`` from the tracker.
cost_stats = proxy.cost_tracker.stats() if proxy.cost_tracker else {}
cost_with = cost_stats.get("cost_with_headroom_usd", 0.0)
compression_savings = cost_stats.get("savings_usd", 0.0)
+ tool_savings = cost_stats.get("tool_savings_usd", 0.0)
cache_net = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
- total_saved_usd = round(compression_savings, 2)
- cost_without = cost_with + compression_savings
+ ext_realized_usd = 0.0
+ ext_projected_usd = 0.0
+ for row in getattr(metrics, "savings_by_source", {}).values():
+ try:
+ usd = float(row.get("usd", 0.0) or 0.0)
+ except (TypeError, ValueError):
+ continue
+ # tokens-only rows (tool_search) carry usd=0 here — the deferral is
+ # priced from the tracker above, so nothing double-counts.
+ if row.get("realized", True):
+ ext_realized_usd += usd
+ else:
+ ext_projected_usd += usd
+ measured_saved = compression_savings + tool_savings + cache_net + ext_realized_usd
+ total_saved_usd = round(measured_saved + ext_projected_usd, 2)
+ cost_without = cost_with + measured_saved + ext_projected_usd
savings_pct_cost = round(total_saved_usd / cost_without * 100, 1) if cost_without > 0 else 0.0
# Primary models used
@@ -629,10 +649,15 @@ def build_session_summary(
"without_headroom_usd": round(cost_without, 2),
"with_headroom_usd": round(cost_with, 2),
"total_saved_usd": total_saved_usd,
+ "measured_saved_usd": round(measured_saved, 2),
+ "projected_saved_usd": round(ext_projected_usd, 2),
"savings_pct": savings_pct_cost,
"breakdown": {
"cache_savings_usd": round(cache_net, 2),
"compression_savings_usd": round(compression_savings, 2),
+ "tool_search_savings_usd": round(tool_savings, 2),
+ "extension_realized_usd": round(ext_realized_usd, 2),
+ "extension_projected_usd": round(ext_projected_usd, 2),
},
},
}
@@ -711,6 +736,19 @@ class CostTracker:
# so the compression-only figure stays available; `stats()` reports both
# the split and the sum.
self._tool_saved_by_model: dict[str, int] = {}
+ # Cache-aware counterfactual buckets for the deferred schemas. Had the
+ # schemas been sent, they would have ridden the SAME prefix as the rest
+ # of the request — cold-written at the provider's cache-write rate,
+ # read at its cache-read rate on warm turns, re-written when the TTL
+ # expired. The observed request's own cache mix is the best available
+ # estimate of that rhythm (a TTL expiry shows up as a write-heavy mix
+ # on the real request, so the counterfactual re-write is captured per
+ # request, not modelled). Priced in stats() via _get_cache_prices —
+ # LiteLLM's per-model catalog — so the rates stay provider-agnostic.
+ # Floats: shares of a request's schema tokens, not whole tokens.
+ self._tool_saved_read_by_model: dict[str, float] = {}
+ self._tool_saved_write_by_model: dict[str, float] = {}
+ self._tool_saved_list_by_model: dict[str, float] = {}
self._tokens_sent_by_model: dict[str, int] = {}
self._requests_by_model: dict[str, int] = {}
@@ -727,6 +765,9 @@ class CostTracker:
self._last_prune_time = datetime.now()
self._tokens_saved_by_model.clear()
self._tool_saved_by_model.clear()
+ self._tool_saved_read_by_model.clear()
+ self._tool_saved_write_by_model.clear()
+ self._tool_saved_list_by_model.clear()
self._tokens_sent_by_model.clear()
self._requests_by_model.clear()
self._api_cache_read_by_model.clear()
@@ -861,6 +902,30 @@ class CostTracker:
self._tool_saved_by_model[model] = self._tool_saved_by_model.get(model, 0) + max(
0, tool_schema_saved
)
+ if tool_schema_saved > 0:
+ # Split this request's deferred schema tokens by the request's own
+ # observed cache mix: the schemas would have shared the prefix, so
+ # they inherit its read/write/uncached proportions. A request the
+ # provider reported no cache data for (or whose write counter was
+ # inferred, not billed) falls back to list price for its full share.
+ write_eff = 0 if cache_inferred else max(0, cache_write_tokens)
+ billed_in = max(0, cache_read_tokens) + write_eff + max(0, uncached_tokens)
+ if billed_in > 0:
+ read_part = tool_schema_saved * max(0, cache_read_tokens) / billed_in
+ write_part = tool_schema_saved * write_eff / billed_in
+ list_part = tool_schema_saved - read_part - write_part
+ else:
+ read_part = write_part = 0.0
+ list_part = float(tool_schema_saved)
+ self._tool_saved_read_by_model[model] = (
+ self._tool_saved_read_by_model.get(model, 0.0) + read_part
+ )
+ self._tool_saved_write_by_model[model] = (
+ self._tool_saved_write_by_model.get(model, 0.0) + write_part
+ )
+ self._tool_saved_list_by_model[model] = (
+ self._tool_saved_list_by_model.get(model, 0.0) + list_part
+ )
self._tokens_sent_by_model[model] = self._tokens_sent_by_model.get(model, 0) + tokens_sent
self._requests_by_model[model] = self._requests_by_model.get(model, 0) + 1
self._api_cache_read_by_model[model] = (
@@ -1186,6 +1251,37 @@ class CostTracker:
_cr_price, _cw_price, uncached_price = prices
savings_usd += saved * uncached_price
+ # Tool-schema deferral, priced CACHE-AWARE rather than flat at list:
+ # had the schemas shipped they would have been part of the same prefix
+ # as the rest of the request — cold-written once at the provider's
+ # cache-write rate, read at its cache-read rate on warm turns, and
+ # re-written on TTL expiry. Each request's schema tokens were bucketed
+ # at record time by that request's observed read/write/uncached mix, so
+ # expiry cycles are captured from real traffic instead of modelled.
+ # Rates come from _get_cache_prices (LiteLLM per-model catalog:
+ # cache_read_input_token_cost / cache_creation_input_token_cost, list
+ # rate when a provider publishes no cache pricing) — the same
+ # provider-agnostic source the real cost math uses, never a hardcoded
+ # multiplier. Reported as its OWN key rather than widening
+ # ``savings_usd``: that figure feeds budget enforcement, and the
+ # session summary widens the cost card from this key instead.
+ tool_savings_usd = 0.0
+ tool_models = (
+ set(self._tool_saved_read_by_model)
+ | set(self._tool_saved_write_by_model)
+ | set(self._tool_saved_list_by_model)
+ )
+ for model in tool_models:
+ prices = self._get_cache_prices(model)
+ if not prices:
+ continue
+ cr_price, cw_price, uncached_price = prices
+ tool_savings_usd += (
+ self._tool_saved_read_by_model.get(model, 0.0) * cr_price
+ + self._tool_saved_write_by_model.get(model, 0.0) * cw_price
+ + self._tool_saved_list_by_model.get(model, 0.0) * uncached_price
+ )
+
return {
# Sum of the per-model rows above, so the payload reconciles with
# itself. ``savings_usd`` below is deliberately NOT widened: tool
@@ -1202,6 +1298,7 @@ class CostTracker:
"per_model": per_model,
"cost_with_headroom_usd": round(cost_with_headroom, 4),
"savings_usd": round(savings_usd, 4),
+ "tool_savings_usd": round(tool_savings_usd, 4),
# Budget config passthrough — surfaces in /stats["cost"] so
# `headroom doctor` can report whether a budget is set.
"budget_limit_usd": self.budget_limit_usd,
diff --git a/headroom/proxy/routing_stats.py b/headroom/proxy/routing_stats.py
new file mode 100644
index 000000000..6e815aec4
--- /dev/null
+++ b/headroom/proxy/routing_stats.py
@@ -0,0 +1,71 @@
+"""Extension seam: model-routing statistics for ``/stats`` and the dashboard.
+
+A proxy extension that rewrites which model serves a request can register a
+provider here; the ``/stats`` payload then carries a ``routing`` section and
+the dashboard renders a "Model Routing" table. Mirrors the lossless-provider
+seam (:mod:`headroom.transforms.lossless_provider`): the core defines the
+contract and renders the data — it does not know who computes it, and it
+stays inert when nothing registers.
+
+Provider contract — return ``None`` (nothing to show) or::
+
+ {
+ "decisions": int, # total routing decisions observed
+ "downgrades": int, # requests actually served on a cheaper model
+ "upgrades": int, # requests actually served on a pricier model
+ "unchanged": int, # decisions that left the request untouched
+ "pairs": [ # requested -> served aggregation
+ {"requested": str, "served": str,
+ "direction": "downgrade" | "upgrade" | "same" | "lateral" | "unknown",
+ "count": int, # decisions for this pair
+ "enforced": int, # of those, how many rewrote the wire
+ "holdout": int, # kept unrouted as a control arm
+ "measured": int, # rows with a measured cost baseline
+ "savings_usd": float}, # signed; negative for upgrades
+ ...
+ ],
+ }
+
+The provider is called on ``/stats`` builds (the dashboard polls a cached
+snapshot), so it should be cheap — an aggregate query, not a table walk.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+
+log = logging.getLogger(__name__)
+
+_provider: Callable[[], dict | None] | None = None
+
+
+def set_routing_stats_provider(fn: Callable[[], dict | None]) -> None:
+ """Register (or replace) the routing-stats provider."""
+ global _provider
+ _provider = fn
+
+
+def clear_routing_stats_provider() -> None:
+ """Test/reset helper."""
+ global _provider
+ _provider = None
+
+
+def get_routing_stats() -> dict | None:
+ """The registered provider's snapshot, or ``None``.
+
+ Never raises: a broken provider must not take down ``/stats``. A payload
+ without a non-empty ``pairs`` list is treated as "nothing to show" so the
+ dashboard has one rule for hiding the section.
+ """
+ if _provider is None:
+ return None
+ try:
+ out = _provider()
+ except Exception:
+ log.debug("routing stats provider failed", exc_info=True)
+ return None
+ if not isinstance(out, dict) or not out.get("pairs"):
+ return None
+ return out
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index d72400d49..923b2af35 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -4311,9 +4311,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
except Exception: # pragma: no cover - defensive
pass
+ # Model-routing section: populated only when an extension registered a
+ # routing-stats provider (headroom.proxy.routing_stats); None otherwise.
+ from headroom.proxy.routing_stats import get_routing_stats
+
return {
"summary": summary,
"agent_usage": agent_usage,
+ "routing": get_routing_stats(),
"savings": {
"total_tokens": total_tokens_all_layers,
"per_project": persistent_savings.get("projects", {}),