Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): aggregate verbosity baselines across projects instead of overwriting (#1288)
gglucass27a546834960
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ae6182f54..4ae691eb2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Bug Fixes+* **learn (verbosity):** `--verbosity --apply --all` now aggregates the savings baseline across every project instead of overwriting it per project (last-project-wins), which previously left the output shaper with a tiny, unrepresentative baseline. The applied verbosity level comes from the project with the most samples ([#1288](https://github.com/headroomlabs-ai/headroom/pull/1288)).* **proxy/anthropic:** restore token-mode compression on continued Claude Code turns with a frozen prefix and deferred CCR tool injection. Token mode now runs request-side compression even when the client did not pre-register `headroom_retrieve`, relying on the existing marker-triggered injection override to keep emitted CCR markers redeemable ([#1487](https://github.com/headroomlabs-ai/headroom/issues/1487)).* **subscription:** stop zeroing the 5-hour headroom contribution counters on every poll. The rollover check compared `five_hour.resets_at` with a bare `!=`, but the usage API reports that timestamp with second-level jitter (observed flapping between `01:59:59Z` and `02:00:00Z` on consecutive polls within the same window), so a spurious "5h window rolled over" reset fired every poll interval (~5 min) and the dashboard's per-window savings stuck near 0%. Only a forward jump larger than `_ROLLOVER_MIN_ADVANCE` (1 minute) now counts as a real rollover.* **transforms/content_router:** stop replacing `role="tool"` output with a lossy-unrecoverable summary on the live compression path (refs [#1307](https://github.com/chopratejas/headroom/issues/1307)). `ContentRouter.apply()` routed OpenAI-style `role="tool"` string messages — `Bash`/`grep`/`ls`/`cat` output — through the ML/word-drop summarizers; when the result carried no CCR retrieve marker (CCR off, ratio >= 0.8, or the size-gate fallback) the original was unrecoverable and the agent acted on a fabricated summary. Tool-role string content is now kept verbatim unless the compressed form is CCR-recoverable. Assistant/user text is unaffected, and structurally-lossless passes (SmartCrusher/Log/Search) still apply. The Anthropic `tool_result` block path is tracked separately.diff --git a/headroom/cli/learn.py b/headroom/cli/learn.pyindex b8a20c405..04cbda325 100644--- a/headroom/cli/learn.py+++ b/headroom/cli/learn.py@@ -170,12 +170,6 @@ def learn(raise click.UsageError("--all and --project are mutually exclusive.")if llm_judge and not verbosity_mode:raise click.UsageError("--llm-judge only applies with --verbosity.")- if verbosity_mode and analyze_all and apply:- raise click.UsageError(- "--verbosity persists a single global level, so --all --apply would keep "- "only the last project's level. Re-run with one --project (or drop --apply "- "to preview every project)."- )max_workers = workers if workers is not None else min(os.cpu_count() or 4, 8)@@ -445,7 +439,7 @@ def _run_verbosity(from ..learn.registry import auto_detect_plugins, get_pluginfrom ..learn.verbosity import analyzefrom ..paths import ensure_workspace_dir- from ..proxy.output_savings import SavingsLedger+ from ..proxy.output_savings import BaselineModel, SavingsLedger# Verbosity mining reads Claude Code transcripts; restrict to that plugin.if agent == "auto":@@ -484,12 +478,26 @@ def _run_verbosity(judge = _make_llm_judge(model or "claude-sonnet-4-6") if llm_judge else None+ # Aggregate across all targeted projects. The baseline accumulates so the+ # synthetic control reflects every project's transcripts (not just whichever+ # one happens to be processed last). The applied verbosity level comes from+ # the project with the most samples — the strongest, least noisy signal.+ aggregated = BaselineModel()+ best_profile = None+ best_profile_samples = -1+ analyzed_count = 0+for proj in targets:session_paths = sorted(proj.data_path.glob("*.jsonl"))if not session_paths:continueprofile, baseline = analyze(session_paths, str(proj.project_path), llm_judge=judge)sig = profile.signals+ analyzed_count += 1+ aggregated.merge(baseline)+ if baseline.total_samples > best_profile_samples:+ best_profile_samples = baseline.total_samples+ best_profile = profileclick.echo(f"\n{'=' * 60}")click.echo(f"Verbosity — {proj.name}")@@ -516,45 +524,49 @@ def _run_verbosity(f"(confidence: {profile.confidence})")- if apply:- ws = ensure_workspace_dir()- from datetime import datetime, timezone-- profile.learned_at = datetime.now(timezone.utc).isoformat()- profile.save(ws / "verbosity.json")- # Seed the savings baseline: replace baseline, preserve any live- # treatment/control already accumulated.- ledger_path = ws / "output_savings.json"- ledger = SavingsLedger.load(ledger_path)- ledger.baseline = baseline- ledger.save(ledger_path)- click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {profile.level})")+ if analyzed_count == 0 or best_profile is None:+ click.echo("\n No transcripts found in the selected project(s); nothing learned.")+ return++ if apply:+ ws = ensure_workspace_dir()+ from datetime import datetime, timezone++ best_profile.learned_at = datetime.now(timezone.utc).isoformat()+ best_profile.save(ws / "verbosity.json")+ # Seed the savings baseline: replace baseline, preserve any live+ # treatment/control already accumulated.+ ledger_path = ws / "output_savings.json"+ ledger = SavingsLedger.load(ledger_path)+ ledger.baseline = aggregated+ ledger.save(ledger_path)+ click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {best_profile.level})")+ click.echo(+ f" [WROTE] {ledger_path} (baseline: {aggregated.total_samples} samples, "+ f"{len(aggregated.strata)} strata across {analyzed_count} project(s))"+ )+ # Writing the level is not enough — the shaper is off by default.+ # Make --apply actually take effect: hot-enable a running proxy, and+ # otherwise tell the user exactly how to turn it on.+ status, shaper_port = _activate_output_shaper()+ if status == "live":click.echo(- f" [WROTE] {ledger_path} (baseline: {baseline.total_samples} samples, "- f"{len(baseline.strata)} strata)"+ f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "+ f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."+ )+ click.echo(+ " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 "+ "before `headroom wrap ...` (wrap pushes it to the proxy).")- # Writing the level is not enough — the shaper is off by default.- # Make --apply actually take effect: hot-enable a running proxy, and- # otherwise tell the user exactly how to turn it on.- status, shaper_port = _activate_output_shaper()- if status == "live":- click.echo(- f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "- f"level {profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."- )- click.echo(- " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 "- "before `headroom wrap ...` (wrap pushes it to the proxy)."- )- else:- click.echo(- "\n ⚠ Level written, but the output shaper is OFF by default — it is "- "NOT shaping output yet."- )- click.echo(- " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` "- "(or start `headroom proxy` with it set). The learned level is then used "- "automatically while HEADROOM_VERBOSITY_LEVEL is unset."- )else:- click.echo("\n Dry run — use --apply to persist the level and baseline.")+ click.echo(+ "\n ⚠ Level written, but the output shaper is OFF by default — it is "+ "NOT shaping output yet."+ )+ click.echo(+ " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` "+ "(or start `headroom proxy` with it set). The learned level is then used "+ "automatically while HEADROOM_VERBOSITY_LEVEL is unset."+ )+ else:+ click.echo("\n Dry run — use --apply to persist the level and baseline.")diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.pyindex b773774c0..bcff07c7e 100644--- a/headroom/proxy/output_savings.py+++ b/headroom/proxy/output_savings.py@@ -162,6 +162,16 @@ class _Accum:return 0.0return max(0.0, (self.sumsq - self.sum * self.sum / self.n) / (self.n - 1))+ def merge(self, other: _Accum) -> None:+ """Fold another accumulator's observations into this one.++ n / sum / sumsq are additive, so merging is element-wise addition and+ is exactly equivalent to having ``add``-ed both observation streams.+ """+ self.n += other.n+ self.sum += other.sum+ self.sumsq += other.sumsq+def to_dict(self) -> dict[str, float]:return {"n": self.n, "sum": self.sum, "sumsq": self.sumsq}@@ -190,6 +200,19 @@ class BaselineModel:self.strata.setdefault(key, _Accum()).add(output_tokens)self.glob.add(output_tokens)+ def merge(self, other: BaselineModel) -> None:+ """Fold another baseline's observations into this one.++ Per-stratum and global accumulators are additive, so merging is+ element-wise and order-independent — the result is identical to having+ observed both corpora against a single model. Used to aggregate a+ cross-project baseline from per-project ``analyze`` results without+ re-reading transcripts.+ """+ for key, acc in other.strata.items():+ self.strata.setdefault(key, _Accum()).merge(acc)+ self.glob.merge(other.glob)+def lookup(self, key: str) -> tuple[float, float, int]:"""Return ``(mean, var, n)`` for *key* with hierarchical back-off.diff --git a/tests/test_cli_learn.py b/tests/test_cli_learn.pyindex b5f58bab9..b160ea8d8 100644--- a/tests/test_cli_learn.py+++ b/tests/test_cli_learn.py@@ -175,6 +175,74 @@ def test_learn_project_lookup_and_apply_flow(assert plugin.writer.calls[0][2] is False+def test_verbosity_all_apply_aggregates_baselines_across_projects(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ import json as _json++ from headroom.proxy.output_savings import BaselineModel, SavingsLedger++ # Two projects, each with a transcript dir holding a dummy session file+ # (analyze is faked, so contents are irrelevant — only presence matters).+ proj_a_dir = tmp_path / "a"+ proj_b_dir = tmp_path / "b"+ for d in (proj_a_dir, proj_b_dir):+ d.mkdir()+ (d / "s.jsonl").write_text("{}")+ proj_a = SimpleNamespace(name="a", project_path=tmp_path / "src-a", data_path=proj_a_dir)+ proj_b = SimpleNamespace(name="b", project_path=tmp_path / "src-b", data_path=proj_b_dir)+ plugin = FakePlugin("claude", "Claude Code", [proj_a, proj_b])++ # Per-project synthetic baselines. Project A has more samples, so its level+ # must be the one applied.+ base_a = BaselineModel()+ for v in (100, 200, 300):+ base_a.observe("opus|new_user_ask|s|tools", v)+ base_b = BaselineModel()+ base_b.observe("sonnet|unknown|m|notools", 50)++ class _Profile:+ def __init__(self, level: int) -> None:+ self.level = level+ self.confidence = "high"+ self.source = "heuristic"+ self.rationale = "test"+ self.signals: dict[str, object] = {}+ self.learned_at: str | None = None++ def save(self, path: object) -> None:+ Path(str(path)).write_text(_json.dumps({"level": self.level}))++ results = {+ str(proj_a.project_path): (_Profile(1), base_a),+ str(proj_b.project_path): (_Profile(3), base_b),+ }++ def fake_analyze(session_paths, project_path, llm_judge=None): # noqa: ANN001, ANN201+ return results[project_path]++ monkeypatch.setattr("headroom.learn.registry.get_plugin", lambda name: plugin)+ monkeypatch.setattr("headroom.learn.verbosity.analyze", fake_analyze)+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))++ result = runner.invoke(+ main,+ ["learn", "--agent", "claude", "--verbosity", "--all", "--apply"],+ catch_exceptions=False,+ )+ assert result.exit_code == 0, result.output++ ledger = SavingsLedger.load(tmp_path / "ws" / "output_savings.json")+ # Aggregated, not last-project-wins: both strata present and totals summed.+ assert ledger.baseline.total_samples == 4+ assert "opus|new_user_ask|s|tools" in ledger.baseline.strata+ assert "sonnet|unknown|m|notools" in ledger.baseline.strata+ assert "across 2 project(s)" in result.output+ # The applied level comes from the project with the most samples (A → 1).+ verbosity = _json.loads((tmp_path / "ws" / "verbosity.json").read_text())+ assert verbosity["level"] == 1++def test_learn_reports_missing_requested_project_and_lists_discovered(monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path) -> None:diff --git a/tests/test_output_savings.py b/tests/test_output_savings.pyindex 6cde64709..fef965687 100644--- a/tests/test_output_savings.py+++ b/tests/test_output_savings.py@@ -129,6 +129,29 @@ class TestBaselineModel:assert m2.lookup("k|a|s|tools") == m.lookup("k|a|s|tools")assert m2.total_samples == 3+ def test_merge_is_equivalent_to_observing_both_streams(self):+ # Merging two baselines must equal observing every value against one+ # model — same totals per stratum and same global fallback.+ a = BaselineModel()+ for v in (100, 200):+ a.observe("opus|new_user_ask|s|tools", v)+ b = BaselineModel()+ b.observe("opus|new_user_ask|s|tools", 300)+ b.observe("sonnet|unknown|m|notools", 50)++ a.merge(b)++ mean, _, n = a.lookup("opus|new_user_ask|s|tools")+ assert n == 3+ assert mean == 200.0 # (100 + 200 + 300) / 3+ assert a.total_samples == 4 # 3 + 1 across both strata++ reference = BaselineModel()+ for v in (100, 200, 300):+ reference.observe("opus|new_user_ask|s|tools", v)+ reference.observe("sonnet|unknown|m|notools", 50)+ assert a.to_dict() == reference.to_dict()+# ---------------------------------------------------------------------------# synthetic-control estimate