Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(learn): aggregate verbosity baselines across projects instead of overwriting (#1288)

gglucass27a546834960
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae6182f54..4ae691eb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Bug Fixes
+* **learn (verbosity):** `--verbosity --apply --all` now aggregates the savings baseline across every project instead of overwriting it per project (last-project-wins), which previously left the output shaper with a tiny, unrepresentative baseline. The applied verbosity level comes from the project with the most samples ([#1288](https://github.com/headroomlabs-ai/headroom/pull/1288)).
* **proxy/anthropic:** restore token-mode compression on continued Claude Code turns with a frozen prefix and deferred CCR tool injection. Token mode now runs request-side compression even when the client did not pre-register `headroom_retrieve`, relying on the existing marker-triggered injection override to keep emitted CCR markers redeemable ([#1487](https://github.com/headroomlabs-ai/headroom/issues/1487)).
* **subscription:** stop zeroing the 5-hour headroom contribution counters on every poll. The rollover check compared `five_hour.resets_at` with a bare `!=`, but the usage API reports that timestamp with second-level jitter (observed flapping between `01:59:59Z` and `02:00:00Z` on consecutive polls within the same window), so a spurious "5h window rolled over" reset fired every poll interval (~5 min) and the dashboard's per-window savings stuck near 0%. Only a forward jump larger than `_ROLLOVER_MIN_ADVANCE` (1 minute) now counts as a real rollover.
* **transforms/content_router:** stop replacing `role="tool"` output with a lossy-unrecoverable summary on the live compression path (refs [#1307](https://github.com/chopratejas/headroom/issues/1307)). `ContentRouter.apply()` routed OpenAI-style `role="tool"` string messages — `Bash`/`grep`/`ls`/`cat` output — through the ML/word-drop summarizers; when the result carried no CCR retrieve marker (CCR off, ratio >= 0.8, or the size-gate fallback) the original was unrecoverable and the agent acted on a fabricated summary. Tool-role string content is now kept verbatim unless the compressed form is CCR-recoverable. Assistant/user text is unaffected, and structurally-lossless passes (SmartCrusher/Log/Search) still apply. The Anthropic `tool_result` block path is tracked separately.
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index b8a20c405..04cbda325 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -170,12 +170,6 @@ def learn(
raise click.UsageError("--all and --project are mutually exclusive.")
if llm_judge and not verbosity_mode:
raise click.UsageError("--llm-judge only applies with --verbosity.")
- if verbosity_mode and analyze_all and apply:
- raise click.UsageError(
- "--verbosity persists a single global level, so --all --apply would keep "
- "only the last project's level. Re-run with one --project (or drop --apply "
- "to preview every project)."
- )
max_workers = workers if workers is not None else min(os.cpu_count() or 4, 8)
@@ -445,7 +439,7 @@ def _run_verbosity(
from ..learn.registry import auto_detect_plugins, get_plugin
from ..learn.verbosity import analyze
from ..paths import ensure_workspace_dir
- from ..proxy.output_savings import SavingsLedger
+ from ..proxy.output_savings import BaselineModel, SavingsLedger
# Verbosity mining reads Claude Code transcripts; restrict to that plugin.
if agent == "auto":
@@ -484,12 +478,26 @@ def _run_verbosity(
judge = _make_llm_judge(model or "claude-sonnet-4-6") if llm_judge else None
+ # Aggregate across all targeted projects. The baseline accumulates so the
+ # synthetic control reflects every project's transcripts (not just whichever
+ # one happens to be processed last). The applied verbosity level comes from
+ # the project with the most samples — the strongest, least noisy signal.
+ aggregated = BaselineModel()
+ best_profile = None
+ best_profile_samples = -1
+ analyzed_count = 0
+
for proj in targets:
session_paths = sorted(proj.data_path.glob("*.jsonl"))
if not session_paths:
continue
profile, baseline = analyze(session_paths, str(proj.project_path), llm_judge=judge)
sig = profile.signals
+ analyzed_count += 1
+ aggregated.merge(baseline)
+ if baseline.total_samples > best_profile_samples:
+ best_profile_samples = baseline.total_samples
+ best_profile = profile
click.echo(f"\n{'=' * 60}")
click.echo(f"Verbosity — {proj.name}")
@@ -516,45 +524,49 @@ def _run_verbosity(
f"(confidence: {profile.confidence})"
)
- if apply:
- ws = ensure_workspace_dir()
- from datetime import datetime, timezone
-
- profile.learned_at = datetime.now(timezone.utc).isoformat()
- profile.save(ws / "verbosity.json")
- # Seed the savings baseline: replace baseline, preserve any live
- # treatment/control already accumulated.
- ledger_path = ws / "output_savings.json"
- ledger = SavingsLedger.load(ledger_path)
- ledger.baseline = baseline
- ledger.save(ledger_path)
- click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {profile.level})")
+ if analyzed_count == 0 or best_profile is None:
+ click.echo("\n No transcripts found in the selected project(s); nothing learned.")
+ return
+
+ if apply:
+ ws = ensure_workspace_dir()
+ from datetime import datetime, timezone
+
+ best_profile.learned_at = datetime.now(timezone.utc).isoformat()
+ best_profile.save(ws / "verbosity.json")
+ # Seed the savings baseline: replace baseline, preserve any live
+ # treatment/control already accumulated.
+ ledger_path = ws / "output_savings.json"
+ ledger = SavingsLedger.load(ledger_path)
+ ledger.baseline = aggregated
+ ledger.save(ledger_path)
+ click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {best_profile.level})")
+ click.echo(
+ f" [WROTE] {ledger_path} (baseline: {aggregated.total_samples} samples, "
+ f"{len(aggregated.strata)} strata across {analyzed_count} project(s))"
+ )
+ # Writing the level is not enough — the shaper is off by default.
+ # Make --apply actually take effect: hot-enable a running proxy, and
+ # otherwise tell the user exactly how to turn it on.
+ status, shaper_port = _activate_output_shaper()
+ if status == "live":
click.echo(
- f" [WROTE] {ledger_path} (baseline: {baseline.total_samples} samples, "
- f"{len(baseline.strata)} strata)"
+ f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "
+ f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."
+ )
+ click.echo(
+ " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 "
+ "before `headroom wrap ...` (wrap pushes it to the proxy)."
)
- # Writing the level is not enough — the shaper is off by default.
- # Make --apply actually take effect: hot-enable a running proxy, and
- # otherwise tell the user exactly how to turn it on.
- status, shaper_port = _activate_output_shaper()
- if status == "live":
- click.echo(
- f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "
- f"level {profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."
- )
- click.echo(
- " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 "
- "before `headroom wrap ...` (wrap pushes it to the proxy)."
- )
- else:
- click.echo(
- "\n ⚠ Level written, but the output shaper is OFF by default — it is "
- "NOT shaping output yet."
- )
- click.echo(
- " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` "
- "(or start `headroom proxy` with it set). The learned level is then used "
- "automatically while HEADROOM_VERBOSITY_LEVEL is unset."
- )
else:
- click.echo("\n Dry run — use --apply to persist the level and baseline.")
+ click.echo(
+ "\n ⚠ Level written, but the output shaper is OFF by default — it is "
+ "NOT shaping output yet."
+ )
+ click.echo(
+ " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` "
+ "(or start `headroom proxy` with it set). The learned level is then used "
+ "automatically while HEADROOM_VERBOSITY_LEVEL is unset."
+ )
+ else:
+ click.echo("\n Dry run — use --apply to persist the level and baseline.")
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py
index b773774c0..bcff07c7e 100644
--- a/headroom/proxy/output_savings.py
+++ b/headroom/proxy/output_savings.py
@@ -162,6 +162,16 @@ class _Accum:
return 0.0
return max(0.0, (self.sumsq - self.sum * self.sum / self.n) / (self.n - 1))
+ def merge(self, other: _Accum) -> None:
+ """Fold another accumulator's observations into this one.
+
+ n / sum / sumsq are additive, so merging is element-wise addition and
+ is exactly equivalent to having ``add``-ed both observation streams.
+ """
+ self.n += other.n
+ self.sum += other.sum
+ self.sumsq += other.sumsq
+
def to_dict(self) -> dict[str, float]:
return {"n": self.n, "sum": self.sum, "sumsq": self.sumsq}
@@ -190,6 +200,19 @@ class BaselineModel:
self.strata.setdefault(key, _Accum()).add(output_tokens)
self.glob.add(output_tokens)
+ def merge(self, other: BaselineModel) -> None:
+ """Fold another baseline's observations into this one.
+
+ Per-stratum and global accumulators are additive, so merging is
+ element-wise and order-independent — the result is identical to having
+ observed both corpora against a single model. Used to aggregate a
+ cross-project baseline from per-project ``analyze`` results without
+ re-reading transcripts.
+ """
+ for key, acc in other.strata.items():
+ self.strata.setdefault(key, _Accum()).merge(acc)
+ self.glob.merge(other.glob)
+
def lookup(self, key: str) -> tuple[float, float, int]:
"""Return ``(mean, var, n)`` for *key* with hierarchical back-off.
diff --git a/tests/test_cli_learn.py b/tests/test_cli_learn.py
index b5f58bab9..b160ea8d8 100644
--- a/tests/test_cli_learn.py
+++ b/tests/test_cli_learn.py
@@ -175,6 +175,74 @@ def test_learn_project_lookup_and_apply_flow(
assert plugin.writer.calls[0][2] is False
+def test_verbosity_all_apply_aggregates_baselines_across_projects(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ import json as _json
+
+ from headroom.proxy.output_savings import BaselineModel, SavingsLedger
+
+ # Two projects, each with a transcript dir holding a dummy session file
+ # (analyze is faked, so contents are irrelevant — only presence matters).
+ proj_a_dir = tmp_path / "a"
+ proj_b_dir = tmp_path / "b"
+ for d in (proj_a_dir, proj_b_dir):
+ d.mkdir()
+ (d / "s.jsonl").write_text("{}")
+ proj_a = SimpleNamespace(name="a", project_path=tmp_path / "src-a", data_path=proj_a_dir)
+ proj_b = SimpleNamespace(name="b", project_path=tmp_path / "src-b", data_path=proj_b_dir)
+ plugin = FakePlugin("claude", "Claude Code", [proj_a, proj_b])
+
+ # Per-project synthetic baselines. Project A has more samples, so its level
+ # must be the one applied.
+ base_a = BaselineModel()
+ for v in (100, 200, 300):
+ base_a.observe("opus|new_user_ask|s|tools", v)
+ base_b = BaselineModel()
+ base_b.observe("sonnet|unknown|m|notools", 50)
+
+ class _Profile:
+ def __init__(self, level: int) -> None:
+ self.level = level
+ self.confidence = "high"
+ self.source = "heuristic"
+ self.rationale = "test"
+ self.signals: dict[str, object] = {}
+ self.learned_at: str | None = None
+
+ def save(self, path: object) -> None:
+ Path(str(path)).write_text(_json.dumps({"level": self.level}))
+
+ results = {
+ str(proj_a.project_path): (_Profile(1), base_a),
+ str(proj_b.project_path): (_Profile(3), base_b),
+ }
+
+ def fake_analyze(session_paths, project_path, llm_judge=None): # noqa: ANN001, ANN201
+ return results[project_path]
+
+ monkeypatch.setattr("headroom.learn.registry.get_plugin", lambda name: plugin)
+ monkeypatch.setattr("headroom.learn.verbosity.analyze", fake_analyze)
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))
+
+ result = runner.invoke(
+ main,
+ ["learn", "--agent", "claude", "--verbosity", "--all", "--apply"],
+ catch_exceptions=False,
+ )
+ assert result.exit_code == 0, result.output
+
+ ledger = SavingsLedger.load(tmp_path / "ws" / "output_savings.json")
+ # Aggregated, not last-project-wins: both strata present and totals summed.
+ assert ledger.baseline.total_samples == 4
+ assert "opus|new_user_ask|s|tools" in ledger.baseline.strata
+ assert "sonnet|unknown|m|notools" in ledger.baseline.strata
+ assert "across 2 project(s)" in result.output
+ # The applied level comes from the project with the most samples (A → 1).
+ verbosity = _json.loads((tmp_path / "ws" / "verbosity.json").read_text())
+ assert verbosity["level"] == 1
+
+
def test_learn_reports_missing_requested_project_and_lists_discovered(
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
) -> None:
diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py
index 6cde64709..fef965687 100644
--- a/tests/test_output_savings.py
+++ b/tests/test_output_savings.py
@@ -129,6 +129,29 @@ class TestBaselineModel:
assert m2.lookup("k|a|s|tools") == m.lookup("k|a|s|tools")
assert m2.total_samples == 3
+ def test_merge_is_equivalent_to_observing_both_streams(self):
+ # Merging two baselines must equal observing every value against one
+ # model — same totals per stratum and same global fallback.
+ a = BaselineModel()
+ for v in (100, 200):
+ a.observe("opus|new_user_ask|s|tools", v)
+ b = BaselineModel()
+ b.observe("opus|new_user_ask|s|tools", 300)
+ b.observe("sonnet|unknown|m|notools", 50)
+
+ a.merge(b)
+
+ mean, _, n = a.lookup("opus|new_user_ask|s|tools")
+ assert n == 3
+ assert mean == 200.0 # (100 + 200 + 300) / 3
+ assert a.total_samples == 4 # 3 + 1 across both strata
+
+ reference = BaselineModel()
+ for v in (100, 200, 300):
+ reference.observe("opus|new_user_ask|s|tools", v)
+ reference.observe("sonnet|unknown|m|notools", 50)
+ assert a.to_dict() == reference.to_dict()
+
# ---------------------------------------------------------------------------
# synthetic-control estimate