Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf: add savings audit output (#1211)
akothari-godaddy9a72bc024263
diff --git a/headroom/cli/perf.py b/headroom/cli/perf.pyindex f92108ebc..148c12e52 100644--- a/headroom/cli/perf.py+++ b/headroom/cli/perf.py@@ -82,8 +82,11 @@ def perf(hours: float, raw: bool, output_format: str) -> None:"tokens_saved","message_tokens_saved","tool_tokens_saved",+ "prompt_reduction_tokens",+ "accounting_delta_tokens","savings_pct","list_price_per_mtok",+ "estimated_list_price_savings_usd",]writer = csv.DictWriter(buf, fieldnames=fieldnames)writer.writeheader()diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.pyindex 999d24050..059cdd9ec 100644--- a/headroom/perf/analyzer.py+++ b/headroom/perf/analyzer.py@@ -613,6 +613,28 @@ def format_report(report: PerfReport) -> str:lines.append(f" · tool schemas {total_tool_saved:,}")lines.append("")+ audit = build_savings_audit(report)+ lines.append("Savings Audit")+ lines.append("-" * 40)+ lines.append(+ " Raw prompt reduction: "+ f"{audit['prompt_reduction_tokens']:,} tokens "+ f"({audit['prompt_reduction_pct']:.1f}%)"+ )+ lines.append(+ " Logged savings: "+ f"{audit['logged_tokens_saved']:,} tokens "+ f"({audit['logged_savings_pct']:.1f}%)"+ )+ lines.append(f" Accounting delta: {audit['accounting_delta_tokens']:+,} tokens")+ delta_count = audit["record_counts"]["with_accounting_delta"]+ if delta_count:+ lines.append(f" Records with delta: {delta_count}/{len(records)}")+ impossible_count = audit["record_counts"]["logged_saved_gt_tokens_before"]+ if impossible_count:+ lines.append(f" Impossible records: {impossible_count} saved more than before")+ lines.append("")+# Per-model breakdown with list pricesby_model: dict[str, list[PerfRecord]] = {}for r in records:@@ -973,6 +995,106 @@ def _pct(saved: int, before: int) -> float:return round(saved / before * 100, 1) if before > 0 else 0.0+def _prompt_reduction_tokens(record: PerfRecord) -> int:+ """Raw prompt-token delta for a record, excluding cache/accounting effects."""+ return max(record.tokens_before - record.tokens_after, 0)+++def _audit_record(record: PerfRecord) -> dict:+ prompt_reduction = _prompt_reduction_tokens(record)+ return {+ "timestamp": record.timestamp,+ "request_id": record.request_id,+ "model": record.model,+ "client": record.client,+ "tokens_before": record.tokens_before,+ "tokens_after": record.tokens_after,+ "logged_tokens_saved": record.tokens_saved,+ "prompt_reduction_tokens": prompt_reduction,+ "accounting_delta_tokens": record.tokens_saved - prompt_reduction,+ "cache_read_tokens": record.cache_read,+ "cache_write_tokens": record.cache_write,+ "optimization_ms": record.optimization_ms,+ "transforms": record.transforms,+ }+++def _record_accounting_reasons(record: PerfRecord) -> list[str]:+ reasons: list[str] = []+ if min(record.tokens_before, record.tokens_after, record.tokens_saved) < 0:+ reasons.append("negative_token_count")+ if record.tokens_before > 0 and record.tokens_saved > record.tokens_before:+ reasons.append("logged_saved_gt_tokens_before")+ if record.tokens_after > record.tokens_before and record.tokens_saved > 0:+ reasons.append("prompt_grew_but_logged_savings_positive")+ if record.tokens_saved != _prompt_reduction_tokens(record):+ reasons.append("logged_saved_ne_prompt_delta")+ return reasons+++def build_savings_audit(report: PerfReport, *, limit: int = 10) -> dict:+ """Explain how headline savings relate to raw before/after token deltas."""+ records = report.perf_records+ total_before = sum(r.tokens_before for r in records)+ total_after = sum(r.tokens_after for r in records)+ logged_saved = sum(r.tokens_saved for r in records)+ prompt_reduction = sum(_prompt_reduction_tokens(r) for r in records)+ prompt_growth = sum(max(r.tokens_after - r.tokens_before, 0) for r in records)+ accounting_delta = logged_saved - prompt_reduction++ records_with_delta = [r for r in records if r.tokens_saved != _prompt_reduction_tokens(r)]+ impossible_saved_gt_before = [+ r for r in records if r.tokens_before > 0 and r.tokens_saved > r.tokens_before+ ]+ prompt_growth_with_savings = [+ r for r in records if r.tokens_after > r.tokens_before and r.tokens_saved > 0+ ]+ negative_token_counts = [+ r for r in records if min(r.tokens_before, r.tokens_after, r.tokens_saved) < 0+ ]++ suspicious_records = []+ for record in sorted(+ records_with_delta,+ key=lambda r: abs(r.tokens_saved - _prompt_reduction_tokens(r)),+ reverse=True,+ )[:limit]:+ suspicious_records.append(+ {+ **_audit_record(record),+ "reasons": _record_accounting_reasons(record),+ }+ )++ return {+ "formula": {+ "raw_prompt_reduction": "sum(max(tokens_before - tokens_after, 0))",+ "logged_savings": "sum(tokens_saved)",+ "accounting_delta": "logged_savings - raw_prompt_reduction",+ },+ "total_tokens_before": total_before,+ "total_tokens_after": total_after,+ "logged_tokens_saved": logged_saved,+ "logged_savings_pct": _pct(logged_saved, total_before),+ "prompt_reduction_tokens": prompt_reduction,+ "prompt_reduction_pct": _pct(prompt_reduction, total_before),+ "prompt_growth_tokens": prompt_growth,+ "accounting_delta_tokens": accounting_delta,+ "record_counts": {+ "total": len(records),+ "with_accounting_delta": len(records_with_delta),+ "logged_saved_gt_tokens_before": len(impossible_saved_gt_before),+ "prompt_grew_but_logged_savings_positive": len(prompt_growth_with_savings),+ "negative_token_count": len(negative_token_counts),+ },+ "top_saving_requests": [+ _audit_record(r)+ for r in sorted(records, key=lambda rec: rec.tokens_saved, reverse=True)[:limit]+ ],+ "suspicious_records": suspicious_records,+ }++def _percentile(data: list[float], pct: float) -> float:if not data:return 0.0@@ -1199,6 +1321,7 @@ def build_perf_summary(report: PerfReport) -> dict:total_before = sum(r.tokens_before for r in records)total_after = sum(r.tokens_after for r in records)total_saved = sum(r.tokens_saved for r in records)+ savings_audit = build_savings_audit(report)total_tool_saved = sum(r.tool_saved for r in records)total_headline_saved = total_saved + total_tool_saved@@ -1211,12 +1334,17 @@ def build_perf_summary(report: PerfReport) -> dict:for r in records:by_model_groups.setdefault(r.model, []).append(r)by_model = []+ estimated_list_price_savings_usd = 0.0for model, recs in sorted(by_model_groups.items()):m_before = sum(r.tokens_before for r in recs)m_after = sum(r.tokens_after for r in recs)m_message_saved = sum(r.tokens_saved for r in recs)m_tool_saved = sum(r.tool_saved for r in recs)m_saved = m_message_saved + m_tool_saved+ m_prompt_reduction = sum(_prompt_reduction_tokens(r) for r in recs)+ list_price = _get_list_price(model)+ if list_price:+ estimated_list_price_savings_usd += m_saved * list_price / 1_000_000by_model.append({"model": model,@@ -1226,8 +1354,13 @@ def build_perf_summary(report: PerfReport) -> dict:"tokens_saved": m_saved,"message_tokens_saved": m_message_saved,"tool_tokens_saved": m_tool_saved,+ "prompt_reduction_tokens": m_prompt_reduction,+ "accounting_delta_tokens": m_message_saved - m_prompt_reduction,"savings_pct": _pct(m_saved, m_before + m_tool_saved),- "list_price_per_mtok": _get_list_price(model),+ "list_price_per_mtok": list_price,+ "estimated_list_price_savings_usd": round(m_saved * list_price / 1_000_000, 4)+ if list_price+ else None,})@@ -1299,6 +1432,8 @@ def build_perf_summary(report: PerfReport) -> dict:"tokens_saved": total_saved,"tool_saved": total_tool_saved,"savings_pct": _pct(total_saved, total_before),+ "estimated_list_price_savings_usd": round(estimated_list_price_savings_usd, 4),+ "savings_audit": savings_audit,"cache_read_tokens": total_cr,"cache_write_tokens": total_cw,"cache_hit_pct": cache_hit_pct,diff --git a/tests/test_cli_perf_format.py b/tests/test_cli_perf_format.pyindex 13dffd7e9..b2dcaf934 100644--- a/tests/test_cli_perf_format.py+++ b/tests/test_cli_perf_format.py@@ -19,6 +19,7 @@ from headroom.perf.analyzer import (TransformRecord,build_overhead_summary,build_perf_summary,+ build_savings_audit,perf_records_as_dicts,)@@ -97,6 +98,8 @@ def test_build_perf_summary_totals_and_pct():assert summary["cache_write_tokens"] == 200assert summary["cache_hit_pct"] == 83.3assert summary["window_hours"] == 24.0+ assert summary["savings_audit"]["prompt_reduction_tokens"] == 1000+ assert summary["savings_audit"]["accounting_delta_tokens"] == 0def test_build_perf_summary_by_model_and_transform():@@ -164,6 +167,40 @@ def test_build_overhead_summary_attributes_slow_stages():assert overhead["top_slow_requests"][0]["slowest_stage"] == "kompress"+def test_build_savings_audit_surfaces_accounting_deltas():+ report = PerfReport(+ perf_records=[+ PerfRecord(+ timestamp="2026-06-05 10:00:00,000",+ request_id="hr_bad",+ model="gpt-5",+ tokens_before=100,+ tokens_after=90,+ tokens_saved=250,+ ),+ PerfRecord(+ timestamp="2026-06-05 10:01:00,000",+ request_id="hr_growth",+ model="gpt-5",+ tokens_before=100,+ tokens_after=120,+ tokens_saved=1,+ ),+ ]+ )++ audit = build_savings_audit(report)++ assert audit["logged_tokens_saved"] == 251+ assert audit["prompt_reduction_tokens"] == 10+ assert audit["accounting_delta_tokens"] == 241+ assert audit["record_counts"]["with_accounting_delta"] == 2+ assert audit["record_counts"]["logged_saved_gt_tokens_before"] == 1+ assert audit["record_counts"]["prompt_grew_but_logged_savings_positive"] == 1+ assert audit["suspicious_records"][0]["request_id"] == "hr_bad"+ assert "logged_saved_gt_tokens_before" in audit["suspicious_records"][0]["reasons"]++def test_perf_records_as_dicts_roundtrips_fields():dicts = perf_records_as_dicts(_sample_report())assert len(dicts) == 2@@ -304,6 +341,8 @@ def test_perf_csv_by_model(runner, monkeypatch):assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")assert sonnet["tokens_saved"] == "600"+ assert sonnet["prompt_reduction_tokens"] == "600"+ assert sonnet["accounting_delta_tokens"] == "0"def test_perf_csv_raw_per_record(runner, monkeypatch):