Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf: add savings audit output (#1211)

akothari-godaddy9a72bc024263
diff --git a/headroom/cli/perf.py b/headroom/cli/perf.py
index f92108ebc..148c12e52 100644
--- a/headroom/cli/perf.py
+++ b/headroom/cli/perf.py
@@ -82,8 +82,11 @@ def perf(hours: float, raw: bool, output_format: str) -> None:
"tokens_saved",
"message_tokens_saved",
"tool_tokens_saved",
+ "prompt_reduction_tokens",
+ "accounting_delta_tokens",
"savings_pct",
"list_price_per_mtok",
+ "estimated_list_price_savings_usd",
]
writer = csv.DictWriter(buf, fieldnames=fieldnames)
writer.writeheader()
diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py
index 999d24050..059cdd9ec 100644
--- a/headroom/perf/analyzer.py
+++ b/headroom/perf/analyzer.py
@@ -613,6 +613,28 @@ def format_report(report: PerfReport) -> str:
lines.append(f" · tool schemas {total_tool_saved:,}")
lines.append("")
+ audit = build_savings_audit(report)
+ lines.append("Savings Audit")
+ lines.append("-" * 40)
+ lines.append(
+ " Raw prompt reduction: "
+ f"{audit['prompt_reduction_tokens']:,} tokens "
+ f"({audit['prompt_reduction_pct']:.1f}%)"
+ )
+ lines.append(
+ " Logged savings: "
+ f"{audit['logged_tokens_saved']:,} tokens "
+ f"({audit['logged_savings_pct']:.1f}%)"
+ )
+ lines.append(f" Accounting delta: {audit['accounting_delta_tokens']:+,} tokens")
+ delta_count = audit["record_counts"]["with_accounting_delta"]
+ if delta_count:
+ lines.append(f" Records with delta: {delta_count}/{len(records)}")
+ impossible_count = audit["record_counts"]["logged_saved_gt_tokens_before"]
+ if impossible_count:
+ lines.append(f" Impossible records: {impossible_count} saved more than before")
+ lines.append("")
+
# Per-model breakdown with list prices
by_model: dict[str, list[PerfRecord]] = {}
for r in records:
@@ -973,6 +995,106 @@ def _pct(saved: int, before: int) -> float:
return round(saved / before * 100, 1) if before > 0 else 0.0
+def _prompt_reduction_tokens(record: PerfRecord) -> int:
+ """Raw prompt-token delta for a record, excluding cache/accounting effects."""
+ return max(record.tokens_before - record.tokens_after, 0)
+
+
+def _audit_record(record: PerfRecord) -> dict:
+ prompt_reduction = _prompt_reduction_tokens(record)
+ return {
+ "timestamp": record.timestamp,
+ "request_id": record.request_id,
+ "model": record.model,
+ "client": record.client,
+ "tokens_before": record.tokens_before,
+ "tokens_after": record.tokens_after,
+ "logged_tokens_saved": record.tokens_saved,
+ "prompt_reduction_tokens": prompt_reduction,
+ "accounting_delta_tokens": record.tokens_saved - prompt_reduction,
+ "cache_read_tokens": record.cache_read,
+ "cache_write_tokens": record.cache_write,
+ "optimization_ms": record.optimization_ms,
+ "transforms": record.transforms,
+ }
+
+
+def _record_accounting_reasons(record: PerfRecord) -> list[str]:
+ reasons: list[str] = []
+ if min(record.tokens_before, record.tokens_after, record.tokens_saved) < 0:
+ reasons.append("negative_token_count")
+ if record.tokens_before > 0 and record.tokens_saved > record.tokens_before:
+ reasons.append("logged_saved_gt_tokens_before")
+ if record.tokens_after > record.tokens_before and record.tokens_saved > 0:
+ reasons.append("prompt_grew_but_logged_savings_positive")
+ if record.tokens_saved != _prompt_reduction_tokens(record):
+ reasons.append("logged_saved_ne_prompt_delta")
+ return reasons
+
+
+def build_savings_audit(report: PerfReport, *, limit: int = 10) -> dict:
+ """Explain how headline savings relate to raw before/after token deltas."""
+ records = report.perf_records
+ total_before = sum(r.tokens_before for r in records)
+ total_after = sum(r.tokens_after for r in records)
+ logged_saved = sum(r.tokens_saved for r in records)
+ prompt_reduction = sum(_prompt_reduction_tokens(r) for r in records)
+ prompt_growth = sum(max(r.tokens_after - r.tokens_before, 0) for r in records)
+ accounting_delta = logged_saved - prompt_reduction
+
+ records_with_delta = [r for r in records if r.tokens_saved != _prompt_reduction_tokens(r)]
+ impossible_saved_gt_before = [
+ r for r in records if r.tokens_before > 0 and r.tokens_saved > r.tokens_before
+ ]
+ prompt_growth_with_savings = [
+ r for r in records if r.tokens_after > r.tokens_before and r.tokens_saved > 0
+ ]
+ negative_token_counts = [
+ r for r in records if min(r.tokens_before, r.tokens_after, r.tokens_saved) < 0
+ ]
+
+ suspicious_records = []
+ for record in sorted(
+ records_with_delta,
+ key=lambda r: abs(r.tokens_saved - _prompt_reduction_tokens(r)),
+ reverse=True,
+ )[:limit]:
+ suspicious_records.append(
+ {
+ **_audit_record(record),
+ "reasons": _record_accounting_reasons(record),
+ }
+ )
+
+ return {
+ "formula": {
+ "raw_prompt_reduction": "sum(max(tokens_before - tokens_after, 0))",
+ "logged_savings": "sum(tokens_saved)",
+ "accounting_delta": "logged_savings - raw_prompt_reduction",
+ },
+ "total_tokens_before": total_before,
+ "total_tokens_after": total_after,
+ "logged_tokens_saved": logged_saved,
+ "logged_savings_pct": _pct(logged_saved, total_before),
+ "prompt_reduction_tokens": prompt_reduction,
+ "prompt_reduction_pct": _pct(prompt_reduction, total_before),
+ "prompt_growth_tokens": prompt_growth,
+ "accounting_delta_tokens": accounting_delta,
+ "record_counts": {
+ "total": len(records),
+ "with_accounting_delta": len(records_with_delta),
+ "logged_saved_gt_tokens_before": len(impossible_saved_gt_before),
+ "prompt_grew_but_logged_savings_positive": len(prompt_growth_with_savings),
+ "negative_token_count": len(negative_token_counts),
+ },
+ "top_saving_requests": [
+ _audit_record(r)
+ for r in sorted(records, key=lambda rec: rec.tokens_saved, reverse=True)[:limit]
+ ],
+ "suspicious_records": suspicious_records,
+ }
+
+
def _percentile(data: list[float], pct: float) -> float:
if not data:
return 0.0
@@ -1199,6 +1321,7 @@ def build_perf_summary(report: PerfReport) -> dict:
total_before = sum(r.tokens_before for r in records)
total_after = sum(r.tokens_after for r in records)
total_saved = sum(r.tokens_saved for r in records)
+ savings_audit = build_savings_audit(report)
total_tool_saved = sum(r.tool_saved for r in records)
total_headline_saved = total_saved + total_tool_saved
@@ -1211,12 +1334,17 @@ def build_perf_summary(report: PerfReport) -> dict:
for r in records:
by_model_groups.setdefault(r.model, []).append(r)
by_model = []
+ estimated_list_price_savings_usd = 0.0
for model, recs in sorted(by_model_groups.items()):
m_before = sum(r.tokens_before for r in recs)
m_after = sum(r.tokens_after for r in recs)
m_message_saved = sum(r.tokens_saved for r in recs)
m_tool_saved = sum(r.tool_saved for r in recs)
m_saved = m_message_saved + m_tool_saved
+ m_prompt_reduction = sum(_prompt_reduction_tokens(r) for r in recs)
+ list_price = _get_list_price(model)
+ if list_price:
+ estimated_list_price_savings_usd += m_saved * list_price / 1_000_000
by_model.append(
{
"model": model,
@@ -1226,8 +1354,13 @@ def build_perf_summary(report: PerfReport) -> dict:
"tokens_saved": m_saved,
"message_tokens_saved": m_message_saved,
"tool_tokens_saved": m_tool_saved,
+ "prompt_reduction_tokens": m_prompt_reduction,
+ "accounting_delta_tokens": m_message_saved - m_prompt_reduction,
"savings_pct": _pct(m_saved, m_before + m_tool_saved),
- "list_price_per_mtok": _get_list_price(model),
+ "list_price_per_mtok": list_price,
+ "estimated_list_price_savings_usd": round(m_saved * list_price / 1_000_000, 4)
+ if list_price
+ else None,
}
)
@@ -1299,6 +1432,8 @@ def build_perf_summary(report: PerfReport) -> dict:
"tokens_saved": total_saved,
"tool_saved": total_tool_saved,
"savings_pct": _pct(total_saved, total_before),
+ "estimated_list_price_savings_usd": round(estimated_list_price_savings_usd, 4),
+ "savings_audit": savings_audit,
"cache_read_tokens": total_cr,
"cache_write_tokens": total_cw,
"cache_hit_pct": cache_hit_pct,
diff --git a/tests/test_cli_perf_format.py b/tests/test_cli_perf_format.py
index 13dffd7e9..b2dcaf934 100644
--- a/tests/test_cli_perf_format.py
+++ b/tests/test_cli_perf_format.py
@@ -19,6 +19,7 @@ from headroom.perf.analyzer import (
TransformRecord,
build_overhead_summary,
build_perf_summary,
+ build_savings_audit,
perf_records_as_dicts,
)
@@ -97,6 +98,8 @@ def test_build_perf_summary_totals_and_pct():
assert summary["cache_write_tokens"] == 200
assert summary["cache_hit_pct"] == 83.3
assert summary["window_hours"] == 24.0
+ assert summary["savings_audit"]["prompt_reduction_tokens"] == 1000
+ assert summary["savings_audit"]["accounting_delta_tokens"] == 0
def test_build_perf_summary_by_model_and_transform():
@@ -164,6 +167,40 @@ def test_build_overhead_summary_attributes_slow_stages():
assert overhead["top_slow_requests"][0]["slowest_stage"] == "kompress"
+def test_build_savings_audit_surfaces_accounting_deltas():
+ report = PerfReport(
+ perf_records=[
+ PerfRecord(
+ timestamp="2026-06-05 10:00:00,000",
+ request_id="hr_bad",
+ model="gpt-5",
+ tokens_before=100,
+ tokens_after=90,
+ tokens_saved=250,
+ ),
+ PerfRecord(
+ timestamp="2026-06-05 10:01:00,000",
+ request_id="hr_growth",
+ model="gpt-5",
+ tokens_before=100,
+ tokens_after=120,
+ tokens_saved=1,
+ ),
+ ]
+ )
+
+ audit = build_savings_audit(report)
+
+ assert audit["logged_tokens_saved"] == 251
+ assert audit["prompt_reduction_tokens"] == 10
+ assert audit["accounting_delta_tokens"] == 241
+ assert audit["record_counts"]["with_accounting_delta"] == 2
+ assert audit["record_counts"]["logged_saved_gt_tokens_before"] == 1
+ assert audit["record_counts"]["prompt_grew_but_logged_savings_positive"] == 1
+ assert audit["suspicious_records"][0]["request_id"] == "hr_bad"
+ assert "logged_saved_gt_tokens_before" in audit["suspicious_records"][0]["reasons"]
+
+
def test_perf_records_as_dicts_roundtrips_fields():
dicts = perf_records_as_dicts(_sample_report())
assert len(dicts) == 2
@@ -304,6 +341,8 @@ def test_perf_csv_by_model(runner, monkeypatch):
assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
assert sonnet["tokens_saved"] == "600"
+ assert sonnet["prompt_reduction_tokens"] == "600"
+ assert sonnet["accounting_delta_tokens"] == "0"
def test_perf_csv_raw_per_record(runner, monkeypatch):