Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(perf): add --format {text,json,csv} to `headroom perf` (#648)

Kumario9fe4886cf6b6
diff --git a/headroom/cli/perf.py b/headroom/cli/perf.py
index 45ea89f2a..992ae9b56 100644
--- a/headroom/cli/perf.py
+++ b/headroom/cli/perf.py
@@ -1,5 +1,9 @@
"""Performance analysis CLI command."""
+import csv
+import io
+import json
+
import click
from .main import main
@@ -13,7 +17,14 @@ from .main import main
help="Analyze logs from the last N hours (default: 168 = 7 days)",
)
@click.option("--raw", is_flag=True, help="Show raw PERF records instead of report")
-def perf(hours: float, raw: bool) -> None:
+ "--format",
+ "output_format",
+ type=click.Choice(["text", "json", "csv"]),
+ default="text",
+ help="Output format (default: text). json/csv emit machine-readable data.",
+)
+def perf(hours: float, raw: bool, output_format: str) -> None:
"""Analyze proxy performance from logs.
\b
@@ -26,14 +37,59 @@ def perf(hours: float, raw: bool) -> None:
\b
Examples:
- headroom perf Analyze last 7 days
- headroom perf --hours 24 Analyze last 24 hours
- headroom perf --raw Show raw parsed records
+ headroom perf Analyze last 7 days
+ headroom perf --hours 24 Analyze last 24 hours
+ headroom perf --raw Show raw parsed records
+ headroom perf --format json Aggregated report as JSON
+ headroom perf --format csv --hours 24 > last-24h.csv
+ headroom perf --format json --raw Raw records as a JSON array
"""
- from headroom.perf.analyzer import format_report, parse_log_files
+ from headroom.perf.analyzer import (
+ PERF_RECORD_FIELDS,
+ build_perf_summary,
+ format_report,
+ parse_log_files,
+ perf_records_as_dicts,
+ )
report = parse_log_files(last_n_hours=hours)
+ if output_format == "json":
+ payload = perf_records_as_dicts(report) if raw else build_perf_summary(report)
+ click.echo(json.dumps(payload, indent=2))
+ return
+
+ if output_format == "csv":
+ buf = io.StringIO()
+ if raw:
+ writer = csv.DictWriter(buf, fieldnames=PERF_RECORD_FIELDS)
+ writer.writeheader()
+ for rec in perf_records_as_dicts(report):
+ row = dict(rec)
+ # Flatten the transforms list for a single CSV cell.
+ row["transforms"] = ",".join(row.get("transforms", []))
+ writer.writerow(row)
+ else:
+ # Non-raw CSV is the per-model breakdown — the most useful tabular
+ # aggregate for spreadsheets and longitudinal charts.
+ summary = build_perf_summary(report)
+ fieldnames = [
+ "model",
+ "requests",
+ "tokens_before",
+ "tokens_after",
+ "tokens_saved",
+ "savings_pct",
+ "list_price_per_mtok",
+ ]
+ writer = csv.DictWriter(buf, fieldnames=fieldnames)
+ writer.writeheader()
+ for row in summary["by_model"]:
+ writer.writerow(row)
+ click.echo(buf.getvalue(), nl=False)
+ return
+
+ # Default: human-readable text.
if raw:
for r in report.perf_records:
click.echo(
diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py
index b52e76ddf..f4df381d0 100644
--- a/headroom/perf/analyzer.py
+++ b/headroom/perf/analyzer.py
@@ -12,7 +12,7 @@ from __future__ import annotations
import logging
import re
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
from datetime import datetime, timedelta
from headroom import paths as _paths
@@ -625,6 +625,128 @@ def format_report(report: PerfReport) -> str:
return "\n".join(lines)
+# ---------------------------------------------------------------------------
+# Machine-readable views (JSON / CSV) — issue #595
+# ---------------------------------------------------------------------------
+#
+# `parse_log_files()` already returns a fully-structured `PerfReport`; these
+# helpers expose it without forcing CI pipelines, dashboards, or agent
+# harnesses to scrape the colored text report. The aggregate numbers mirror
+# `format_report()` exactly so a JSON consumer and a human reading the report
+# never disagree.
+
+# Column order for the per-record (`--raw`) machine output. Kept as a module
+# constant so the CLI's CSV writer and any external consumer share one source
+# of truth.
+PERF_RECORD_FIELDS = [
+ "timestamp",
+ "request_id",
+ "model",
+ "num_messages",
+ "tokens_before",
+ "tokens_after",
+ "tokens_saved",
+ "cache_read",
+ "cache_write",
+ "cache_hit_pct",
+ "optimization_ms",
+ "transforms",
+]
+
+
+def _pct(saved: int, before: int) -> float:
+ """Reduction percentage, rounded to 1dp, guarding divide-by-zero."""
+ return round(saved / before * 100, 1) if before > 0 else 0.0
+
+
+def build_perf_summary(report: PerfReport) -> dict:
+ """Aggregate a ``PerfReport`` into a JSON-serialisable summary dict.
+
+ The shape mirrors the human-readable ``format_report`` numbers so the same
+ data drives CI regression guards (``jq '.savings_pct < 70'``), dashboards,
+ and end-of-session savings summaries in agent wrappers.
+ """
+ records = report.perf_records
+
+ total_before = sum(r.tokens_before for r in records)
+ total_after = sum(r.tokens_after for r in records)
+ total_saved = sum(r.tokens_saved for r in records)
+
+ total_cr = sum(r.cache_read for r in records)
+ total_cw = sum(r.cache_write for r in records)
+ total_cache = total_cr + total_cw
+ cache_hit_pct = round(total_cr / total_cache * 100, 1) if total_cache > 0 else 0.0
+
+ by_model_groups: dict[str, list[PerfRecord]] = {}
+ for r in records:
+ by_model_groups.setdefault(r.model, []).append(r)
+ by_model = []
+ for model, recs in sorted(by_model_groups.items()):
+ m_before = sum(r.tokens_before for r in recs)
+ m_after = sum(r.tokens_after for r in recs)
+ m_saved = sum(r.tokens_saved for r in recs)
+ by_model.append(
+ {
+ "model": model,
+ "requests": len(recs),
+ "tokens_before": m_before,
+ "tokens_after": m_after,
+ "tokens_saved": m_saved,
+ "savings_pct": _pct(m_saved, m_before),
+ "list_price_per_mtok": _get_list_price(model),
+ }
+ )
+
+ by_transform_groups: dict[str, list[TransformRecord]] = {}
+ for tr in report.transform_records:
+ by_transform_groups.setdefault(tr.name, []).append(tr)
+ by_transform = []
+ for name, t_recs in sorted(
+ by_transform_groups.items(), key=lambda kv: -sum(r.tokens_saved for r in kv[1])
+ ):
+ t_before = sum(r.tokens_before for r in t_recs)
+ t_saved = sum(r.tokens_saved for r in t_recs)
+ by_transform.append(
+ {
+ "transform": name,
+ "uses": len(t_recs),
+ "tokens_before": t_before,
+ "tokens_saved": t_saved,
+ "savings_pct": _pct(t_saved, t_before),
+ }
+ )
+
+ return {
+ "window_hours": report.requested_hours,
+ "actual_window": {
+ "oldest": report.oldest_kept_ts,
+ "newest": report.newest_kept_ts,
+ },
+ "records_filtered_out": report.records_filtered_out,
+ "total_requests": len(records),
+ "total_tokens_before": total_before,
+ "total_tokens_after": total_after,
+ "tokens_saved": total_saved,
+ "savings_pct": _pct(total_saved, total_before),
+ "cache_read_tokens": total_cr,
+ "cache_write_tokens": total_cw,
+ "cache_hit_pct": cache_hit_pct,
+ "by_model": by_model,
+ "by_transform": by_transform,
+ "log_files_read": report.log_files_read,
+ "total_lines_parsed": report.total_lines_parsed,
+ }
+
+
+def perf_records_as_dicts(report: PerfReport) -> list[dict]:
+ """Per-record view of the parsed PERF entries (for ``--raw`` machine output).
+
+ ``transforms`` stays a list so JSON consumers keep structure; the CSV
+ writer flattens it to a comma-joined string at the edge.
+ """
+ return [asdict(r) for r in report.perf_records]
+
+
def _format_toin_highlights() -> list[str]:
"""Render a human-readable TOIN highlights block from the live store.
diff --git a/tests/test_cli_perf_format.py b/tests/test_cli_perf_format.py
new file mode 100644
index 000000000..6ef9875fb
--- /dev/null
+++ b/tests/test_cli_perf_format.py
@@ -0,0 +1,190 @@
+"""Tests for `headroom perf --format {text,json,csv}` (issue #595)."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from headroom.cli.main import main
+from headroom.perf import analyzer
+from headroom.perf.analyzer import (
+ PerfRecord,
+ PerfReport,
+ TransformRecord,
+ build_perf_summary,
+ perf_records_as_dicts,
+)
+
+
+def runner() -> CliRunner:
+ return CliRunner()
+
+
+def _sample_report() -> PerfReport:
+ """A small report with two models, cache numbers, and a transform."""
+ return PerfReport(
+ perf_records=[
+ PerfRecord(
+ timestamp="2026-06-05 10:00:00,000",
+ request_id="hr_1",
+ model="claude-sonnet-4.5",
+ num_messages=10,
+ tokens_before=1000,
+ tokens_after=400,
+ tokens_saved=600,
+ cache_read=800,
+ cache_write=200,
+ cache_hit_pct=80,
+ optimization_ms=12.0,
+ transforms=["content_router"],
+ ),
+ PerfRecord(
+ timestamp="2026-06-05 11:00:00,000",
+ request_id="hr_2",
+ model="claude-opus-4-8",
+ num_messages=4,
+ tokens_before=1000,
+ tokens_after=600,
+ tokens_saved=400,
+ cache_read=200,
+ cache_write=0,
+ cache_hit_pct=100,
+ optimization_ms=8.0,
+ transforms=["content_router"],
+ ),
+ ],
+ transform_records=[
+ TransformRecord(
+ timestamp="2026-06-05 10:00:00,000",
+ name="content_router",
+ tokens_before=2000,
+ tokens_after=1000,
+ tokens_saved=1000,
+ ),
+ ],
+ log_files_read=1,
+ total_lines_parsed=42,
+ requested_hours=24.0,
+ oldest_kept_ts="2026-06-05 10:00:00,000",
+ newest_kept_ts="2026-06-05 11:00:00,000",
+ )
+
+
+# ---------------------------------------------------------------------------
+# Pure builders
+# ---------------------------------------------------------------------------
+
+
+def test_build_perf_summary_totals_and_pct():
+ summary = build_perf_summary(_sample_report())
+
+ assert summary["total_requests"] == 2
+ assert summary["total_tokens_before"] == 2000
+ assert summary["total_tokens_after"] == 1000
+ assert summary["tokens_saved"] == 1000
+ # 1000 / 2000 == 50.0%
+ assert summary["savings_pct"] == 50.0
+ # cache: read 1000, write 200 -> 1000 / 1200 == 83.3%
+ assert summary["cache_read_tokens"] == 1000
+ assert summary["cache_write_tokens"] == 200
+ assert summary["cache_hit_pct"] == 83.3
+ assert summary["window_hours"] == 24.0
+
+
+def test_build_perf_summary_by_model_and_transform():
+ summary = build_perf_summary(_sample_report())
+
+ models = {m["model"]: m for m in summary["by_model"]}
+ assert set(models) == {"claude-sonnet-4.5", "claude-opus-4-8"}
+ assert models["claude-sonnet-4.5"]["tokens_saved"] == 600
+ assert models["claude-sonnet-4.5"]["savings_pct"] == 60.0
+ assert models["claude-opus-4-8"]["savings_pct"] == 40.0
+
+ assert summary["by_transform"][0]["transform"] == "content_router"
+ assert summary["by_transform"][0]["tokens_saved"] == 1000
+ assert summary["by_transform"][0]["uses"] == 1
+
+
+def test_build_perf_summary_empty_report_no_zero_division():
+ summary = build_perf_summary(PerfReport(requested_hours=168.0))
+ assert summary["total_requests"] == 0
+ assert summary["savings_pct"] == 0.0
+ assert summary["cache_hit_pct"] == 0.0
+ assert summary["by_model"] == []
+
+
+def test_perf_records_as_dicts_roundtrips_fields():
+ dicts = perf_records_as_dicts(_sample_report())
+ assert len(dicts) == 2
+ assert dicts[0]["request_id"] == "hr_1"
+ assert dicts[0]["tokens_saved"] == 600
+ # transforms stays a list for JSON consumers
+ assert dicts[0]["transforms"] == ["content_router"]
+
+
+# ---------------------------------------------------------------------------
+# CLI integration
+# ---------------------------------------------------------------------------
+
+
+def _patch_report(monkeypatch, report: PerfReport) -> None:
+ monkeypatch.setattr(analyzer, "parse_log_files", lambda last_n_hours=168.0: report)
+
+
+def test_perf_json_format(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf", "--format", "json"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.output)
+ assert data["savings_pct"] == 50.0
+ assert "by_model" in data
+ assert data["total_requests"] == 2
+
+
+def test_perf_json_raw_is_array(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf", "--format", "json", "--raw"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.output)
+ assert isinstance(data, list)
+ assert len(data) == 2
+ assert data[0]["request_id"] == "hr_1"
+
+
+def test_perf_csv_by_model(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf", "--format", "csv"])
+ assert result.exit_code == 0, result.output
+ rows = list(csv.DictReader(io.StringIO(result.output)))
+ assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
+ sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
+ assert sonnet["tokens_saved"] == "600"
+
+
+def test_perf_csv_raw_per_record(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf", "--format", "csv", "--raw"])
+ assert result.exit_code == 0, result.output
+ rows = list(csv.DictReader(io.StringIO(result.output)))
+ assert len(rows) == 2
+ assert rows[0]["request_id"] == "hr_1"
+ # transforms flattened to a string cell
+ assert rows[0]["transforms"] == "content_router"
+
+
+def test_perf_text_default_unchanged(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf"])
+ assert result.exit_code == 0, result.output
+ assert "Headroom Performance Report" in result.output
+
+
+def test_perf_rejects_unknown_format(runner, monkeypatch):
+ _patch_report(monkeypatch, _sample_report())
+ result = runner.invoke(main, ["perf", "--format", "xml"])
+ assert result.exit_code != 0