Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(dashboard): per-model savings breakdown and expected-vs-actual cost on historical charts (#807)
Focused Instability34dafe69d907
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.htmlindex a2efe9369..f1e088a47 100644--- a/headroom/dashboard/templates/dashboard.html+++ b/headroom/dashboard/templates/dashboard.html@@ -1110,13 +1110,23 @@<span class="text-sm font-medium text-gray-300">Historical Savings Trend</span><span class="text-xs text-gray-500 font-mono" x-text="historyTrendLabel"></span></div>- <div class="inline-flex rounded-lg border border-border p-1 self-start lg:self-auto" style="background: var(--card-alt-bg);">- <template x-for="[label, key] in historyGranularityOptions" :key="key">- <button class="px-3 py-1.5 text-sm rounded-md transition-colors"- :class="historyGranularity === key ? 'bg-accent text-black' : 'text-gray-400 hover:text-gray-200'"- @click="historyGranularity = key"- x-text="label"></button>- </template>+ <div class="flex flex-wrap gap-2 self-start lg:self-auto">+ <div class="inline-flex rounded-lg border border-border p-1" style="background: var(--card-alt-bg);">+ <template x-for="[label, key] in historyGranularityOptions" :key="key">+ <button class="px-3 py-1.5 text-sm rounded-md transition-colors"+ :class="historyGranularity === key ? 'bg-accent text-black' : 'text-gray-400 hover:text-gray-200'"+ @click="historyGranularity = key"+ x-text="label"></button>+ </template>+ </div>+ <div class="inline-flex rounded-lg border border-border p-1" style="background: var(--card-alt-bg);">+ <template x-for="[label, key] in historyChartModeOptions" :key="key">+ <button class="px-3 py-1.5 text-sm rounded-md transition-colors"+ :class="historyChartMode === key ? 'bg-accent text-black' : 'text-gray-400 hover:text-gray-200'"+ @click="historyChartMode = key"+ x-text="label"></button>+ </template>+ </div></div></div><div class="text-xs text-gray-500 mb-4"@@ -1130,10 +1140,24 @@<stop offset="100%" style="stop-color:#22d3ee;stop-opacity:0"/></linearGradient></defs>- <path :d="getObjectTrendArea(historicalTrend, 'total_tokens_saved')"+ <path :d="historyChartMode === 'tokens' && !historyActiveModel ? getObjectTrendArea(historicalTrend, 'total_tokens_saved') : ''"fill="url(#history-trend-gradient)"></path><path class="trend-line"- :d="getObjectTrendLine(historicalTrend, 'total_tokens_saved')"></path>+ :d="historyChartMode === 'tokens' && !historyActiveModel ? getObjectTrendLine(historicalTrend, 'total_tokens_saved') : ''"></path>+ <path fill="none" stroke-width="1"+ :stroke="historyModelColor(0)" :d="historyModelLine(0)"></path>+ <path fill="none" stroke-width="1"+ :stroke="historyModelColor(1)" :d="historyModelLine(1)"></path>+ <path fill="none" stroke-width="1"+ :stroke="historyModelColor(2)" :d="historyModelLine(2)"></path>+ <path fill="none" stroke-width="1"+ :stroke="historyModelColor(3)" :d="historyModelLine(3)"></path>+ <path fill="none" stroke-width="1"+ :stroke="historyModelColor(4)" :d="historyModelLine(4)"></path>+ <path fill="none" stroke="#22d3ee" stroke-width="1.5"+ :d="historyCostLine('actual')"></path>+ <path fill="none" stroke="#fbbf24" stroke-width="1.5" stroke-dasharray="3 2"+ :d="historyCostLine('expected')"></path></svg></div></template>@@ -1142,6 +1166,81 @@Historical trend data will appear after more saved checkpoints in this granularity.</div></template>+ <div class="flex flex-wrap gap-3 mt-3"+ x-show="historyChartMode === 'tokens' && historyGranularity !== 'history' && historyModelChartSeries.length > 0">+ <template x-for="(series, index) in historyModelChartSeries" :key="series.model">+ <button class="flex items-center gap-1.5 text-xs transition-opacity cursor-pointer"+ :class="historyActiveModel && historyActiveModel !== series.model+ ? 'text-gray-600 opacity-50'+ : 'text-gray-400 hover:text-gray-200'"+ @click="toggleHistoryModel(series.model)"+ :title="historySelectedModel === series.model+ ? 'Show all models'+ : 'Show only this model'">+ <span class="inline-block w-2 h-2 rounded-full"+ :style="'background:' + historyModelColor(index)"></span>+ <span :class="historySelectedModel === series.model ? 'underline' : ''"+ x-text="truncateModel(series.model)"></span>+ </button>+ </template>+ </div>+ <div class="flex flex-wrap gap-4 mt-3" x-show="historyChartMode === 'cost'">+ <span class="flex items-center gap-1.5 text-xs text-gray-400">+ <span class="inline-block w-3 h-0.5" style="background:#22d3ee"></span>+ Actual cost (with Headroom)+ </span>+ <span class="flex items-center gap-1.5 text-xs text-gray-400">+ <span class="inline-block w-3 h-0.5"+ style="background:repeating-linear-gradient(90deg,#fbbf24 0 4px,transparent 4px 7px)"></span>+ Expected cost (without Headroom)+ </span>+ </div>+ </div>++ <div class="bg-surface rounded-lg p-4 border border-border">+ <div class="flex items-center justify-between mb-4">+ <span class="text-sm font-medium text-gray-300">Per-Model Breakdown</span>+ <span class="text-xs text-gray-500 font-mono"+ x-text="historyModelSourceSeriesLabel + ' buckets'"></span>+ </div>+ <template x-if="historyModelBreakdown.length === 0">+ <div class="text-sm text-gray-500 italic">+ Per-model attribution appears for checkpoints recorded after upgrading.+ </div>+ </template>+ <template x-if="historyModelBreakdown.length > 0">+ <div class="overflow-x-auto">+ <table class="w-full text-sm">+ <thead>+ <tr class="text-xs text-gray-500 text-left">+ <th class="py-2 pr-4 font-medium">Model</th>+ <th class="py-2 pr-4 font-medium text-right">Tokens saved</th>+ <th class="py-2 pr-4 font-medium text-right">Cost with Headroom</th>+ <th class="py-2 pr-4 font-medium text-right">Expected cost without Headroom</th>+ <th class="py-2 font-medium text-right">Saved</th>+ </tr>+ </thead>+ <tbody>+ <template x-for="row in historyModelBreakdown" :key="row.model">+ <tr class="border-t border-border text-gray-300 cursor-pointer transition-colors"+ :class="historySelectedModel === row.model ? 'bg-[#1c1c1c]' : 'hover:bg-[#181818]'"+ @click="toggleHistoryModel(row.model)"+ :title="historySelectedModel === row.model+ ? 'Show all models'+ : 'Show only this model in the chart'">+ <td class="py-2 pr-4 font-mono text-xs"+ :class="historySelectedModel === row.model ? 'underline' : ''"+ x-text="truncateModel(row.model)"></td>+ <td class="py-2 pr-4 text-right tabular-nums" x-text="formatNumber(row.tokens_saved)"></td>+ <td class="py-2 pr-4 text-right tabular-nums" x-text="'$' + formatCurrency(row.input_cost_usd)"></td>+ <td class="py-2 pr-4 text-right tabular-nums" x-text="'$' + formatCurrency(row.expected_cost_usd)"></td>+ <td class="py-2 text-right tabular-nums text-accent" x-text="'$' + formatCurrency(row.savings_usd)"></td>+ </tr>+ </template>+ </tbody>+ </table>+ </div>+ </template></div><div class="bg-surface rounded-lg p-4 border border-border">@@ -1358,6 +1457,8 @@lastUpdate: 'never',viewMode: 'session',historyGranularity: 'daily',+ historyChartMode: 'tokens',+ historySelectedModel: null,requestHistory: [],savingsHistory: [],expandedRows: {},@@ -1673,6 +1774,151 @@];},+ get historyChartModeOptions() {+ return [+ ['Tokens', 'tokens'],+ ['Cost', 'cost'],+ ];+ },++ get historyModelSourceSeries() {+ // Rollup buckets carrying by_model attribution. Raw checkpoints+ // have no by_model, so the checkpoint view falls back to daily.+ const key = this.historySelectedSeriesKey === 'history'+ ? 'daily'+ : this.historySelectedSeriesKey;+ return this.historyStats.series?.[key] || [];+ },++ get historyModelChartSeries() {+ const buckets = this.historyModelSourceSeries;+ const totals = {};+ for (const bucket of buckets) {+ for (const [model, entry] of Object.entries(bucket.by_model || {})) {+ totals[model] = (totals[model] || 0) + (entry.tokens_saved || 0);+ }+ }+ const topModels = Object.entries(totals)+ .filter(([, saved]) => saved > 0)+ .sort((a, b) => b[1] - a[1])+ .slice(0, 5)+ .map(([model]) => model);+ // A breakdown-row selection outside the top 5 takes the+ // last chart slot so the filter works for every row (the+ // template renders a fixed set of line slots).+ const selected = this.historySelectedModel;+ if (+ selected &&+ (totals[selected] || 0) > 0 &&+ topModels.length > 0 &&+ !topModels.includes(selected)+ ) {+ topModels[topModels.length - 1] = selected;+ }+ return topModels.map(model => {+ let running = 0;+ return {+ model,+ values: buckets.map(bucket => {+ running += bucket.by_model?.[model]?.tokens_saved || 0;+ return running;+ }),+ };+ });+ },++ get historyModelBreakdown() {+ const totals = {};+ for (const bucket of this.historyModelSourceSeries) {+ for (const [model, entry] of Object.entries(bucket.by_model || {})) {+ const row = totals[model] || (totals[model] = {+ model,+ tokens_saved: 0,+ savings_usd: 0,+ input_cost_usd: 0,+ });+ row.tokens_saved += entry.tokens_saved || 0;+ row.savings_usd += entry.compression_savings_usd_delta || 0;+ row.input_cost_usd += entry.total_input_cost_usd_delta || 0;+ }+ }+ return Object.values(totals)+ .map(row => ({+ ...row,+ expected_cost_usd: row.input_cost_usd + row.savings_usd,+ }))+ .sort((a, b) => b.tokens_saved - a.tokens_saved);+ },++ get historyActiveModel() {+ // A model filter only applies while that model is present in+ // the charted series; otherwise fall back to showing all.+ // Raw checkpoint view plots no per-model lines (they are+ // derived from rollup buckets), so the filter must not+ // suppress the aggregate line there.+ if (this.historySelectedSeriesKey === 'history') return null;+ const model = this.historySelectedModel;+ if (!model) return null;+ return this.historyModelChartSeries.some(series => series.model === model)+ ? model+ : null;+ },++ toggleHistoryModel(model) {+ this.historySelectedModel = this.historySelectedModel === model ? null : model;+ },++ get historyCostTrend() {+ return this.historicalTrend.map(point => {+ const actual = point?.total_input_cost_usd || 0;+ const saved = point?.compression_savings_usd || 0;+ return { actual, expected: actual + saved };+ });+ },++ historyModelColor(index) {+ const palette = ['#a78bfa', '#34d399', '#fbbf24', '#f87171', '#60a5fa'];+ return palette[index % palette.length];+ },++ historyModelLine(index) {+ if (this.historyChartMode !== 'tokens') return '';+ // Checkpoint view plots raw checkpoints; daily-derived model+ // lines would not share its x-axis.+ if (this.historySelectedSeriesKey === 'history') return '';+ const allSeries = this.historyModelChartSeries;+ const series = allSeries[index];+ if (!series || series.values.length < 2) return '';+ const activeModel = this.historyActiveModel;+ if (activeModel && series.model !== activeModel) return '';+ // A single filtered model gets its own scale; the full set+ // shares one so the lines stay comparable.+ const scaleSeries = activeModel ? [series] : allSeries;+ const max = Math.max(...scaleSeries.flatMap(s => s.values), 1);+ return this.buildTrendPath(series.values, 0, max);+ },++ historyCostLine(kind) {+ if (this.historyChartMode !== 'cost') return '';+ const trend = this.historyCostTrend;+ if (trend.length < 2) return '';+ const all = trend.flatMap(point => [point.actual, point.expected]);+ const min = Math.min(...all);+ const max = Math.max(...all);+ return this.buildTrendPath(trend.map(point => point[kind]), min, max);+ },++ buildTrendPath(values, min, max) {+ if (!values || values.length < 2) return '';+ const range = max - min || 1;+ const points = values.map((value, index) => {+ const x = (index / (values.length - 1)) * 200;+ const y = 60 - ((value - min) / range) * 56;+ return `${x},${y}`;+ });+ return 'M' + points.join(' L');+ },+get hasHistoricalData() {return (this.historyStats.history || []).length > 0;},@@ -1691,6 +1937,14 @@return labels[this.historySelectedSeriesKey] || 'History';},+ get historyModelSourceSeriesLabel() {+ // historyModelSourceSeries substitutes the daily rollup+ // at raw checkpoint granularity; label what is shown.+ return this.historySelectedSeriesKey === 'history'+ ? 'Daily'+ : this.historySelectedSeriesLabel;+ },+get historySelectedPointCount() {if (this.historySelectedSeriesKey === 'history') {return (this.historyStats.history || []).length;diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex 6a7ebfba2..cbeab8fad 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -133,6 +133,22 @@ def _normalize_provider(value: Any) -> str:return cleaned or PROVIDER_UNKNOWN+MODEL_UNKNOWN = "unknown"+++def _normalize_model(value: Any) -> str:+ """Normalize a model label, falling back to a stable sentinel.++ History checkpoints persisted before per-model attribution existed have+ no model field, so they collapse into ``MODEL_UNKNOWN`` rather than+ silently dropping their savings from the per-model breakdown.+ """+ if not isinstance(value, str):+ return MODEL_UNKNOWN+ cleaned = value.strip()+ return cleaned or MODEL_UNKNOWN++def _resolve_litellm_model(model: str) -> str:"""Resolve model name to one LiteLLM recognizes."""litellm = _get_litellm_module()@@ -243,6 +259,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:total_input_tokens = 0total_input_cost_usd = 0.0provider = PROVIDER_UNKNOWN+ model = MODEL_UNKNOWNif isinstance(entry, dict):timestamp = _parse_timestamp(entry.get("timestamp"))@@ -251,6 +268,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:total_input_tokens = _coerce_int(entry.get("total_input_tokens"))total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))provider = _normalize_provider(entry.get("provider"))+ model = _normalize_model(entry.get("model"))elif isinstance(entry, list | tuple) and len(entry) >= 2:timestamp = _parse_timestamp(entry[0])total_tokens_saved = _coerce_int(entry[1])@@ -269,6 +287,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:return {"timestamp": _to_utc_iso(timestamp),"provider": provider,+ "model": model,"total_tokens_saved": total_tokens_saved,"compression_savings_usd": round(compression_savings_usd, 6),"total_input_tokens": total_input_tokens,@@ -470,6 +489,7 @@ class SavingsTracker:{"timestamp": _to_utc_iso(timestamp_dt),"provider": _normalize_provider(provider),+ "model": _normalize_model(model),"total_tokens_saved": lifetime["tokens_saved"],"compression_savings_usd": lifetime["compression_savings_usd"],"total_input_tokens": lifetime["total_input_tokens"],@@ -602,6 +622,7 @@ class SavingsTracker:{"timestamp": _to_utc_iso(timestamp_dt),"provider": _normalize_provider(provider),+ "model": _normalize_model(model),"total_tokens_saved": lifetime["tokens_saved"],"compression_savings_usd": lifetime["compression_savings_usd"],"total_input_tokens": lifetime["total_input_tokens"],@@ -1077,6 +1098,7 @@ class SavingsTracker:"total_input_cost_usd_delta": 0.0,"total_input_cost_usd": total_input_cost_usd,"by_provider": {},+ "by_model": {},},)entry["tokens_saved"] += delta_tokens@@ -1120,4 +1142,25 @@ class SavingsTracker:6,)+ model = _normalize_model(point.get("model"))+ mod = entry["by_model"].setdefault(+ model,+ {+ "tokens_saved": 0,+ "compression_savings_usd_delta": 0.0,+ "total_input_tokens_delta": 0,+ "total_input_cost_usd_delta": 0.0,+ },+ )+ mod["tokens_saved"] += delta_tokens+ mod["compression_savings_usd_delta"] = round(+ mod["compression_savings_usd_delta"] + delta_usd,+ 6,+ )+ mod["total_input_tokens_delta"] += delta_input_tokens+ mod["total_input_cost_usd_delta"] = round(+ mod["total_input_cost_usd_delta"] + delta_input_cost_usd,+ 6,+ )+return list(aggregated.values())diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex 421505d8c..6e634a391 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -66,6 +66,7 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc) == {"timestamp": "2026-03-27T09:00:00Z","provider": "unknown",+ "model": "unknown","total_tokens_saved": 12,"compression_savings_usd": 0.5,"total_input_tokens": 0,@@ -124,6 +125,7 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):{"timestamp": "2026-03-27T09:00:00Z","provider": "unknown",+ "model": "unknown","total_tokens_saved": 30,"compression_savings_usd": 0.03,"total_input_tokens": 0,@@ -193,6 +195,7 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp{"timestamp": "2026-03-27T08:00:00Z","provider": "unknown",+ "model": "gpt-4o","total_tokens_saved": 10,"compression_savings_usd": 0.01,"total_input_tokens": 120,@@ -201,6 +204,7 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp{"timestamp": "2026-03-27T12:34:00Z","provider": "unknown",+ "model": "gpt-4o","total_tokens_saved": 15,"compression_savings_usd": 0.015,"total_input_tokens": 180,@@ -603,6 +607,119 @@ def test_savings_tracker_rollup_attributes_savings_per_provider(tmp_path, monkeyassert third["by_provider"]["unknown"]["tokens_saved"] == 15+def test_savings_tracker_rollup_attributes_savings_per_model(tmp_path, monkeypatch):+ path = tmp_path / "proxy_savings.json"+ tracker = SavingsTracker(+ path=str(path),+ max_history_points=100,+ max_history_age_days=30,+ )++ monkeypatch.setattr(+ "headroom.proxy.savings_tracker._estimate_compression_savings_usd",+ lambda model, tokens_saved: tokens_saved / 1000.0,+ )++ # Two models from the same provider land in the same bucket.+ tracker.record_compression_savings(+ model="claude-sonnet-4-6",+ tokens_saved=100,+ provider="anthropic",+ total_input_tokens=120,+ total_input_cost_usd=0.24,+ timestamp="2026-03-27T09:10:00Z",+ )+ tracker.record_compression_savings(+ model="claude-opus-4-8",+ tokens_saved=40,+ provider="anthropic",+ total_input_tokens=200,+ total_input_cost_usd=0.40,+ timestamp="2026-03-27T09:40:00Z",+ )+ tracker.record_compression_savings(+ model="claude-sonnet-4-6",+ tokens_saved=25,+ provider="anthropic",+ total_input_tokens=260,+ total_input_cost_usd=0.52,+ timestamp="2026-03-27T10:05:00Z",+ )++ response = tracker.history_response()++ # Checkpoints persist the model alongside the provider.+ assert [point["model"] for point in response["history"]] == [+ "claude-sonnet-4-6",+ "claude-opus-4-8",+ "claude-sonnet-4-6",+ ]++ hourly = response["series"]["hourly"]++ first = hourly[0]+ assert set(first["by_model"]) == {"claude-sonnet-4-6", "claude-opus-4-8"}+ assert first["by_model"]["claude-sonnet-4-6"]["tokens_saved"] == 100+ assert first["by_model"]["claude-sonnet-4-6"]["total_input_tokens_delta"] == 120+ assert first["by_model"]["claude-sonnet-4-6"]["compression_savings_usd_delta"] == pytest.approx(+ 0.1+ )+ assert first["by_model"]["claude-sonnet-4-6"]["total_input_cost_usd_delta"] == pytest.approx(+ 0.24+ )+ assert first["by_model"]["claude-opus-4-8"]["tokens_saved"] == 40+ # Per-model deltas sum back to the bucket total.+ assert (+ first["by_model"]["claude-sonnet-4-6"]["tokens_saved"]+ + first["by_model"]["claude-opus-4-8"]["tokens_saved"]+ == first["tokens_saved"]+ )++ second = hourly[1]+ assert set(second["by_model"]) == {"claude-sonnet-4-6"}+ assert second["by_model"]["claude-sonnet-4-6"]["tokens_saved"] == 25++ # The expected no-headroom cost is derivable per bucket: actual input cost+ # delta plus the compression savings delta.+ sonnet = first["by_model"]["claude-sonnet-4-6"]+ assert sonnet["total_input_cost_usd_delta"] + sonnet["compression_savings_usd_delta"] == (+ pytest.approx(0.34)+ )+++def test_legacy_checkpoints_without_model_collapse_into_unknown(tmp_path):+ path = tmp_path / "proxy_savings.json"+ legacy_state = {+ "schema_version": 2,+ "lifetime": {+ "requests": 1,+ "tokens_saved": 50,+ "compression_savings_usd": 0.05,+ "total_input_tokens": 100,+ "total_input_cost_usd": 0.2,+ },+ "history": [+ {+ "timestamp": "2026-03-27T09:10:00Z",+ "provider": "anthropic",+ "total_tokens_saved": 50,+ "compression_savings_usd": 0.05,+ "total_input_tokens": 100,+ "total_input_cost_usd": 0.2,+ }+ ],+ }+ path.write_text(json.dumps(legacy_state), encoding="utf-8")++ tracker = SavingsTracker(path=str(path))+ response = tracker.history_response()++ assert response["history"][0]["model"] == "unknown"+ hourly = response["series"]["hourly"]+ assert set(hourly[0]["by_model"]) == {"unknown"}+ assert hourly[0]["by_model"]["unknown"]["tokens_saved"] == 50++def test_stats_history_defaults_to_compact_history_but_can_return_full_history(tmp_path, monkeypatch):@@ -826,3 +943,14 @@ def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):assert "Export CSV" in htmlassert "Weekly Savings" in htmlassert "Monthly Savings" in html+ assert "Per-Model Breakdown" in html+ assert "historyChartModeOptions" in html+ assert "Expected cost (without Headroom)" in html+ assert "toggleHistoryModel" in html+ # Checkpoint view plots no per-model lines, so an active model+ # filter must not suppress the aggregate line there.+ assert "if (this.historySelectedSeriesKey === 'history') return null;" in html+ # Breakdown header labels the effective (substituted) series.+ assert "historyModelSourceSeriesLabel + ' buckets'" in html+ # Non-top-5 breakdown rows swap into the last chart slot when selected.+ assert "topModels[topModels.length - 1] = selected;" in html