Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): pin the learned verbosity level on a cache-mode proxy (#4033)
Kenneth Wong71663b44e26f
diff --git a/README.md b/README.mdindex c9561ff9d..a478cf7f8 100644--- a/README.md+++ b/README.md@@ -211,9 +211,14 @@ picks the level:```bashheadroom learn --verbosity # dry run — preview what it found-headroom learn --verbosity --apply # save it; the proxy picks it up+headroom learn --verbosity --apply # save it and apply it to the running proxy```+A proxy in cache mode (the default) never re-reads the learned level, because a+level that changes mid-conversation would bust the prompt cache. There `--apply`+pins it with `HEADROOM_VERBOSITY_LEVEL`; set that variable yourself to keep it+after a restart.+**Measuring it.** Output savings are counterfactual — we never see what the model*would* have written — so Headroom reports an estimate with a confidence rangeand labels it as one:diff --git a/headroom/cli/learn.py b/headroom/cli/learn.pyindex da1303816..47269c11f 100644--- a/headroom/cli/learn.py+++ b/headroom/cli/learn.py@@ -3,7 +3,7 @@from __future__ import annotationsfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, NamedTupleimport click@@ -418,7 +418,167 @@ def _make_llm_judge(model: str) -> Any:return judge-def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:+def _local_proxy_port(port: int | None = None) -> int:+ import os as _os++ return port if port is not None else int(_os.environ.get("HEADROOM_PORT", "8787"))+++def _steered_level(raw: object) -> int | None:+ """The level a proxy steers at for a raw ``HEADROOM_VERBOSITY_LEVEL``.++ Mirrors ``OutputShaperSettings.from_env``: unparseable text falls back to+ the default level, anything else is clamped to 0-4. ``None`` when unset.+ """+ from ..proxy.output_shaper import DEFAULT_VERBOSITY_LEVEL++ if raw is None or str(raw).strip() == "":+ return None+ try:+ level = int(str(raw).strip())+ except ValueError:+ level = DEFAULT_VERBOSITY_LEVEL+ return max(0, min(4, level))+++class _ProxyProcess(NamedTuple):+ """Process identity and its raw verbosity pin from the same health snapshot."""++ pid: int+ started_at: float+ verbosity_pin: str | None = None+++def _proxy_process(payload: dict[str, Any], config: dict[str, Any]) -> _ProxyProcess | None:+ """The proxy process behind a ``/health`` payload, or ``None`` if unknown.++ The start time is ``timestamp - uptime_seconds``: the proxy computes both+ from ``time.time()`` while building the payload, so the difference is its+ start time to the millisecond, the same on every call.+ """+ from datetime import datetime++ pid = config.get("pid")+ timestamp = payload.get("timestamp")+ uptime = payload.get("uptime_seconds")+ if not isinstance(pid, int) or not isinstance(timestamp, str):+ return None+ # A proxy that has not recorded its start reports 0.0; that is not an identity.+ if not isinstance(uptime, int | float) or uptime <= 0:+ return None+ try:+ now = datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp()+ except ValueError:+ return None+ runtime = config.get("runtime_env")+ raw_pin = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None+ return _ProxyProcess(+ pid, round(now - float(uptime), 3), raw_pin if isinstance(raw_pin, str) else None+ )+++def _query_proxy_verbosity(+ port: int | None = None,+) -> tuple[str | None, int | None, _ProxyProcess | None]:+ """Best-effort: read a running local proxy's mode, pinned level and process.++ All three come from ``/health``, whose config block the proxy includes for+ loopback callers. Returns ``(mode, level, process)``, each ``None`` when the+ proxy is unreachable or does not report it; ``level`` is the level its+ ``HEADROOM_VERBOSITY_LEVEL`` pins, ``None`` when that is unset.+ """+ import json as _json+ import urllib.error+ import urllib.request++ url = f"http://127.0.0.1:{_local_proxy_port(port)}/health"+ try:+ with urllib.request.urlopen(url, timeout=2) as response:+ payload = _json.loads(response.read() or b"{}")+ except (urllib.error.URLError, OSError, ValueError):+ return None, None, None+ config = payload.get("config") if isinstance(payload, dict) else None+ if not isinstance(config, dict):+ return None, None, None+ mode = config.get("mode")+ runtime = config.get("runtime_env")+ level = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None+ return (+ mode if isinstance(mode, str) else None,+ _steered_level(level),+ _proxy_process(payload, config),+ )+++#: Records the HEADROOM_VERBOSITY_LEVEL that ``--apply`` hot-pinned, and on+#: which proxy process. Hot overrides die with the process, so a pin is ours to+#: replace only while that same process still holds that value.+_VERBOSITY_PIN_FILE = "verbosity_pin.json"++#: Slack when comparing recorded and current start times, which agree to the+#: millisecond for one process; a different process started this close to the+#: old one would also need its pid, port and level to match.+_START_TIME_TOLERANCE_S = 1.0+++def _pin_is_ours(ws: Path, port: int, process: _ProxyProcess | None, level: int | None) -> bool:+ import json as _json++ if process is None or level is None:+ return False+ try:+ record = _json.loads((ws / _VERBOSITY_PIN_FILE).read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return False+ if not isinstance(record, dict):+ return False+ started_at = record.get("started_at")+ return (+ record.get("port") == port+ and record.get("pid") == process.pid+ and record.get("level") == level+ and process.verbosity_pin == str(level)+ and isinstance(started_at, int | float)+ and abs(started_at - process.started_at) <= _START_TIME_TOLERANCE_S+ )+++def _record_pin(ws: Path, port: int, process: _ProxyProcess | None, level: int) -> bool:+ """Record a pin ``--apply`` set. Returns False when it could not be recorded.++ Best-effort: the pin is already live on the proxy, so a failed record only+ means a later ``--apply`` treats it as the operator's and leaves it alone.+ """+ import json as _json++ path = ws / _VERBOSITY_PIN_FILE+ try:+ if process is None:+ # Without a process identity the pin cannot be attributed later.+ path.unlink(missing_ok=True)+ return False+ record = {+ "port": port,+ "pid": process.pid,+ "started_at": process.started_at,+ "level": level,+ }+ path.write_text(_json.dumps(record), encoding="utf-8", newline="\n")+ except OSError:+ return False+ return True+++class _ShaperActivation(NamedTuple):+ status: str+ port: int+ #: The overrides the proxy reports it applied (empty if it did not answer).+ applied: dict[str, str]+++def _activate_output_shaper(+ port: int | None = None, *, verbosity_level: str | None = None+) -> _ShaperActivation:"""Best-effort: turn the output shaper ON for a running local proxy.Writing ``verbosity.json`` is inert on its own — the shaper is a live,@@ -426,20 +586,24 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:``HEADROOM_OUTPUT_SHAPER`` is enabled in the proxy that serves traffic.When a proxy is already running locally we hot-enable it via``/admin/runtime-env`` (no restart, the same channel ``wrap`` uses), so- ``--apply`` actually takes effect. Returns ``(status, port)`` where status is- ``"live"`` (enabled on a running proxy), ``"blocked"`` (the proxy's- rollout channel rejected it), ``"absent"`` (no reachable proxy), or+ ``--apply`` actually takes effect. ``verbosity_level`` also pins+ ``HEADROOM_VERBOSITY_LEVEL`` in the same request. Returns ``(status, port,+ applied)`` where status is ``"live"`` (enabled on a running proxy),+ ``"blocked"`` (the proxy's rollout config disables the shaper; the proxy+ still applies the overrides), ``"absent"`` (no reachable proxy), or``"error"``."""import json as _json- import os as _osimport urllib.errorimport urllib.request- resolved_port = port if port is not None else int(_os.environ.get("HEADROOM_PORT", "8787"))+ resolved_port = _local_proxy_port(port)+ overrides = {"HEADROOM_OUTPUT_SHAPER": "1"}+ if verbosity_level is not None:+ overrides["HEADROOM_VERBOSITY_LEVEL"] = verbosity_levelrequest = urllib.request.Request(f"http://127.0.0.1:{resolved_port}/admin/runtime-env",- data=_json.dumps({"HEADROOM_OUTPUT_SHAPER": "1"}).encode("utf-8"),+ data=_json.dumps(overrides).encode("utf-8"),method="POST",headers={"Content-Type": "application/json"},)@@ -447,6 +611,12 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:with urllib.request.urlopen(request, timeout=2) as response:raw_response = response.read()payload = _json.loads(raw_response) if raw_response else {}+ applied_raw = payload.get("applied") if isinstance(payload, dict) else None+ applied = (+ {k: v for k, v in applied_raw.items() if isinstance(v, str)}+ if isinstance(applied_raw, dict)+ else {}+ )rollout = payload.get("rollout") if isinstance(payload, dict) else Noneif isinstance(rollout, dict):decisions = rollout.get("features")@@ -460,13 +630,13 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:None,)if isinstance(output_shaper, dict) and not output_shaper.get("enabled", False):- return "blocked", resolved_port- return "live", resolved_port+ return _ShaperActivation("blocked", resolved_port, applied)+ return _ShaperActivation("live", resolved_port, applied)except (urllib.error.URLError, OSError):# ConnectionRefused (no proxy) or 404 (proxy predates the endpoint).- return "absent", resolved_port+ return _ShaperActivation("absent", resolved_port, {})except ValueError:- return "error", resolved_port+ return _ShaperActivation("error", resolved_port, {})def _run_verbosity(@@ -593,24 +763,79 @@ def _run_verbosity(# Writing the level is not enough — the shaper is off by default.# Make --apply actually take effect: hot-enable a running proxy, and# otherwise tell the user exactly how to turn it on.- status, shaper_port = _activate_output_shaper()- if status == "live":+ level = best_profile.level+ port = _local_proxy_port()+ mode, pinned, process = _query_proxy_verbosity(port)+ # Cache mode never reads verbosity.json: a level that can change while+ # conversations are open would bust their prefix cache, so the proxy+ # steers at its startup level unless HEADROOM_VERBOSITY_LEVEL pins one+ # (resolve_verbosity_level). Pin the learned level there; otherwise the+ # proxy would keep steering at its default while we report this level.+ # An existing pin is the operator's and is left alone, unless the record+ # from an earlier --apply shows this same proxy process holds the value+ # that run pinned.+ ours = _pin_is_ours(ws, port, process, pinned)+ pin = str(level) if mode == "cache" and (pinned is None or ours) else None+ status, shaper_port, applied = _activate_output_shaper(port, verbosity_level=pin)+ # Record the pin whenever the proxy applied it: it stores overrides even+ # when its rollout config disables the shaper ("blocked").+ pin_applied = pin is not None and applied.get("HEADROOM_VERBOSITY_LEVEL") == pin+ pin_recorded = pin_applied and _record_pin(ws, shaper_port, process, level)+ cache_mode_hint = (+ f"in cache mode (the default) also set HEADROOM_VERBOSITY_LEVEL={level}, "+ "because cache mode does not read the learned level"+ )+ if status == "live" and pinned is not None and pin is None and pinned != level:+ click.echo(+ f"\n ⚠ Output shaper enabled on the running proxy (port {shaper_port}), "+ f"but it stays at level {pinned}: HEADROOM_VERBOSITY_LEVEL is set there "+ "and outranks the learned level."+ )+ click.echo(+ f" To use level {level}, restart the proxy with HEADROOM_VERBOSITY_LEVEL={level}."+ )+ elif status == "live" and mode is None:+ click.echo(+ f"\n ⚠ Output shaper enabled on the running proxy (port {shaper_port}), but "+ f"its mode could not be read, so level {level} may not be in use."+ )+ click.echo(+ f" A proxy in cache mode (the default) keeps its startup level; restart it "+ f"with HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL={level} to be sure."+ )+ elif status == "live":+ how = (+ " (pinned with HEADROOM_VERBOSITY_LEVEL: cache mode does not read the "+ "learned level)"+ if pin+ else ""+ )click.echo(f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "- f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."+ f"level {level} is live now{how}."+ )+ if pin and pinned != level:+ click.echo(+ " Conversations already open re-cache their prompt once, because "+ "the steering text changed."+ )+ restart_env = "HEADROOM_OUTPUT_SHAPER=1" + (+ f" HEADROOM_VERBOSITY_LEVEL={level}" if mode == "cache" else "")click.echo(- " To keep it on across restarts: export HEADROOM_ROLLOUT_CHANNEL=beta "- "and HEADROOM_OUTPUT_SHAPER=1 before `headroom wrap ...`."+ f" To keep it on across restarts: export {restart_env} "+ "before `headroom wrap ...` or `headroom proxy`.")elif status == "blocked":+ # proxy_output_shaper is available on every channel, so only an+ # explicit disable (HEADROOM_DISABLE_FEATURES) refuses the hot enable.click.echo(- "\n ⚠ Level written, but the running proxy's rollout channel blocks the "- "beta output shaper."+ "\n ⚠ Level written, but the running proxy's rollout config disables the "+ "output shaper.")click.echo(- " Restart it with HEADROOM_ROLLOUT_CHANNEL=beta and "- "HEADROOM_OUTPUT_SHAPER=1; the learned level will be used automatically."+ " Restart it without proxy_output_shaper in HEADROOM_DISABLE_FEATURES "+ f"and with HEADROOM_OUTPUT_SHAPER=1; {cache_mode_hint}.")else:click.echo(@@ -618,10 +843,15 @@ def _run_verbosity("NOT shaping output yet.")click.echo(- " Enable it: export HEADROOM_ROLLOUT_CHANNEL=beta and "- "HEADROOM_OUTPUT_SHAPER=1, then run `headroom wrap ...` (or restart "- "`headroom proxy`). The learned level is then used automatically while "- "HEADROOM_VERBOSITY_LEVEL is unset."+ " Enable it: export HEADROOM_OUTPUT_SHAPER=1, then run `headroom wrap ...` "+ "(or restart `headroom proxy`). A token-mode proxy then uses the learned "+ f"level; {cache_mode_hint}."+ )+ if pin_applied and not pin_recorded:+ click.echo(+ f" Note: the HEADROOM_VERBOSITY_LEVEL={level} pin could not be recorded in "+ f"{ws / _VERBOSITY_PIN_FILE}, so a later --apply will leave it in place as "+ "if you had set it.")else:click.echo("\n Dry run — use --apply to persist the level and baseline.")diff --git a/tests/test_cli_learn.py b/tests/test_cli_learn.pyindex 8ebc7e858..95a52d078 100644--- a/tests/test_cli_learn.py+++ b/tests/test_cli_learn.py@@ -330,6 +330,8 @@ def test_verbosity_all_apply_aggregates_baselines_across_projects(monkeypatch.setattr("headroom.learn.registry.get_plugin", lambda name: plugin)monkeypatch.setattr("headroom.learn.verbosity.analyze", fake_analyze)monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))+ # --apply talks to a local proxy; keep it off whatever is listening on 8787.+ monkeypatch.setattr("urllib.request.urlopen", _no_local_proxy)result = runner.invoke(main,@@ -344,6 +346,9 @@ def test_verbosity_all_apply_aggregates_baselines_across_projects(assert "opus|new_user_ask|s|tools" in ledger.baseline.strataassert "sonnet|unknown|m|notools" in ledger.baseline.strataassert "across 2 project(s)" in result.output+ # The shaper is available on every channel; the hint must not ask for beta.+ assert "HEADROOM_OUTPUT_SHAPER=1" in result.output+ assert "HEADROOM_ROLLOUT_CHANNEL" not in result.output# The applied level comes from the project with the most samples (A → 1).verbosity = _json.loads((tmp_path / "ws" / "verbosity.json").read_text())assert verbosity["level"] == 1@@ -637,7 +642,7 @@ def test_activate_output_shaper_reports_effective_rollout_decision(monkeypatch.setattr(urllib.request, "urlopen", lambda *args, **kwargs: Response())- status, port = _activate_output_shaper(9876)+ status, port, _ = _activate_output_shaper(9876)assert status == expectedassert port == 9876@@ -662,4 +667,432 @@ def test_activate_output_shaper_handles_malformed_response(monkeypatch.setattr(urllib.request, "urlopen", lambda *args, **kwargs: Response())- assert _activate_output_shaper(9876) == ("error", 9876)+ assert _activate_output_shaper(9876) == ("error", 9876, {})+++def _no_local_proxy(*args, **kwargs): # noqa: ANN002, ANN003, ANN201+ import urllib.error++ raise urllib.error.URLError("no proxy in tests")+++# Where the stubbed local proxy listens; HEADROOM_PORT points the CLI at it.+_PROXY_PORT = 18787+# The fake proxy's start time and the wall clock it answers /health at.+_STARTED_AT = 1_791_400_000.0+_HEALTH_NOW = "2026-10-08T10:00:00Z"+++class _FakeProxy:+ """A local proxy as ``urllib.request.urlopen`` sees it.++ ``health_config`` is the ``/health`` config block (``None`` = unreachable);+ POSTs to ``/admin/runtime-env`` are recorded and accepted.+ """++ def __init__(+ self,+ health_config: dict | None,+ *,+ shaper_allowed: bool = True,+ started_at: float = _STARTED_AT,+ ) -> None:+ self.health_config = health_config+ self.shaper_allowed = shaper_allowed+ self.started_at = started_at+ self.posted: list[dict] = []++ def urlopen(self, request, timeout=None): # noqa: ANN001, ANN201+ import io+ import urllib.error++ url = request if isinstance(request, str) else request.full_url+ if url.endswith("/health"):+ if self.health_config is None:+ raise urllib.error.URLError("unreachable")+ from datetime import datetime++ now = datetime.fromisoformat(_HEALTH_NOW.replace("Z", "+00:00")).timestamp()+ payload = {+ "timestamp": _HEALTH_NOW,+ "uptime_seconds": round(now - self.started_at, 3),+ "config": self.health_config,+ }+ return io.BytesIO(json.dumps(payload).encode())+ self.posted.append(json.loads(request.data))+ rollout = {"features": [{"name": "proxy_output_shaper", "enabled": self.shaper_allowed}]}+ return io.BytesIO(json.dumps({"applied": self.posted[-1], "rollout": rollout}).encode())+++def _apply_learned_level(+ monkeypatch: pytest.MonkeyPatch,+ runner: CliRunner,+ tmp_path: Path,+ proxy: _FakeProxy,+ *,+ level: int = 3,+ pin_record: dict | list | None = None,+ previous_level: int | None = None,+) -> str:+ """Run ``learn --verbosity --apply`` for one project that learns ``level``.++ ``pin_record`` seeds the record an earlier ``--apply`` leaves after pinning,+ ``previous_level`` the verbosity.json it saved. The proxy listens on+ ``_PROXY_PORT``.+ """+ from headroom.proxy.output_savings import BaselineModel++ workspace = tmp_path / "ws"+ workspace.mkdir(exist_ok=True)+ if pin_record is not None:+ (workspace / "verbosity_pin.json").write_text(json.dumps(pin_record), newline="\n")+ if previous_level is not None:+ (workspace / "verbosity.json").write_text(+ json.dumps({"verbosity_level": previous_level}), newline="\n"+ )+ data_dir = tmp_path / "sessions"+ data_dir.mkdir()+ (data_dir / "s.jsonl").write_text("{}")+ project = SimpleNamespace(+ name="p", project_path=tmp_path / "src", data_path=data_dir, extra_data_paths=[]+ )+ baseline = BaselineModel()+ baseline.observe("opus|new_user_ask|s|tools", 100)+ profile = SimpleNamespace(+ level=level,+ confidence="high",+ source="heuristic",+ rationale="test",+ signals={},+ learned_at=None,+ save=lambda path: Path(str(path)).write_text(+ json.dumps({"verbosity_level": level}), newline="\n"+ ),+ )+ monkeypatch.setattr(+ "headroom.learn.registry.get_plugin", lambda name: FakePlugin(name, "Claude", [project])+ )+ monkeypatch.setattr(+ "headroom.learn.verbosity.analyze",+ lambda session_paths, project_path, llm_judge=None: (profile, baseline),+ )+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(workspace))+ monkeypatch.setenv("HEADROOM_PORT", str(_PROXY_PORT))+ monkeypatch.setattr("urllib.request.urlopen", proxy.urlopen)++ result = runner.invoke(+ main,+ ["learn", "--agent", "claude", "--verbosity", "--all", "--apply"],+ catch_exceptions=False,+ )+ assert result.exit_code == 0, result.output+ return result.output+++def test_verbosity_apply_pins_the_level_on_a_cache_mode_proxy(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """Cache mode never reads verbosity.json, so "live" needs the level pinned."""+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": None}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]+ assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output+ assert "HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL=3 before" in output+ assert "re-cache their prompt once" in output+++def test_verbosity_apply_leaves_a_token_mode_proxy_reading_the_profile(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """Token mode reads verbosity.json per request; a pin would only shadow it."""+ proxy = _FakeProxy({"mode": "token", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": None}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "level 3 is live now." in output+ assert "export HEADROOM_OUTPUT_SHAPER=1 before" in output+++def test_verbosity_apply_does_not_override_an_explicit_level(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "2"}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "is live now" not in output+ assert "it stays at level 2: HEADROOM_VERBOSITY_LEVEL is set there" in output+++def test_verbosity_apply_records_the_pin_it_sets(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})++ _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3}+++def test_verbosity_apply_replaces_the_pin_an_earlier_apply_set(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """A re-learned level must not be shadowed by the previous run's own pin."""+ proxy = _FakeProxy(+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}+ )++ output = _apply_learned_level(+ monkeypatch,+ runner,+ tmp_path,+ proxy,+ level=1,+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3},+ )++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "1"}]+ assert "level 1 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output+ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 1}+++def test_verbosity_apply_re_pinning_the_same_level_does_not_warn_about_re_caching(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy(+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}+ )++ output = _apply_learned_level(+ monkeypatch,+ runner,+ tmp_path,+ proxy,+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3},+ )++ assert "level 3 is live now" in output+ assert "re-cache" not in output+++ "pin_record",+ [+ None, # the operator pinned the level that was learned before+ # pinned on a proxy since restarted+ {"port": _PROXY_PORT, "pid": 1111, "started_at": _STARTED_AT, "level": 3},+ # a new process that reused the old one's pid on the same port+ {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT - 3600, "level": 3},+ # the operator re-pinned after us+ {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 2},+ # a record from before start times were recorded+ {"port": _PROXY_PORT, "pid": 4242, "level": 3},+ # a record that is valid JSON but not an object+ [_PROXY_PORT, 4242, _STARTED_AT, 3],+ ],+ ids=[+ "no-record",+ "restarted-proxy",+ "reused-pid",+ "changed-since",+ "no-start-time",+ "not-an-object",+ ],+)+def test_verbosity_apply_keeps_a_pin_it_cannot_prove_it_set(+ monkeypatch: pytest.MonkeyPatch,+ runner: CliRunner,+ tmp_path: Path,+ pin_record: dict | list | None,+) -> None:+ proxy = _FakeProxy(+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}+ )++ output = _apply_learned_level(+ monkeypatch, runner, tmp_path, proxy, level=1, pin_record=pin_record, previous_level=3+ )++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "it stays at level 3: HEADROOM_VERBOSITY_LEVEL is set there" in output++[email protected]_newline+def test_verbosity_pin_record_is_written_with_lf(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001+ from headroom.cli import learn as learn_cli++ calls: list[dict] = []+ real_write_text = Path.write_text++ def write_text(self, data, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202+ calls.append(kwargs)+ return real_write_text(self, data, *args, **kwargs)++ monkeypatch.setattr(Path, "write_text", write_text)+ process = learn_cli._ProxyProcess(4242, _STARTED_AT)+ assert learn_cli._record_pin(tmp_path, _PROXY_PORT, process, 3) is True++ assert [call.get("newline") for call in calls] == ["\n"]++[email protected]("raw", ["9", " 04 "])+def test_verbosity_apply_reads_a_pin_the_way_the_proxy_clamps_it(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path, raw: str+) -> None:+ """The proxy steers a pin of 9 (or " 04 ") at level 4, so it matches a learned 4."""+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": raw}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=4)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "level 4 is live now." in output+ assert "stays at level" not in output+++def test_verbosity_apply_reads_an_unparseable_pin_as_the_default_level(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """The proxy steers a non-numeric pin at the default level, and so must the CLI."""+ from headroom.proxy.output_shaper import DEFAULT_VERBOSITY_LEVEL++ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "high"}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=3)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert f"it stays at level {DEFAULT_VERBOSITY_LEVEL}: HEADROOM_VERBOSITY_LEVEL" in output+++def test_verbosity_apply_does_not_claim_live_when_the_mode_is_unknown(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy(None)++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "is live now" not in output+ assert "its mode could not be read" in output+++def test_verbosity_apply_on_a_proxy_that_disables_the_shaper(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}}, shaper_allowed=False)++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert "rollout config disables the output shaper" in output+ assert "HEADROOM_DISABLE_FEATURES" in output+ assert "HEADROOM_VERBOSITY_LEVEL=3" in output+ assert "HEADROOM_ROLLOUT_CHANNEL" not in output+++def test_verbosity_apply_records_a_pin_the_proxy_applied_while_the_shaper_is_disabled(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """The proxy stores the pin even when it reports the shaper blocked; own it."""+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242}, shaper_allowed=False)++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]+ assert "rollout config disables the output shaper" in output+ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3}+++def test_verbosity_apply_reports_activation_when_the_pin_cannot_be_recorded(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})+ # A directory where the record file should go makes the write fail.+ (tmp_path / "ws" / "verbosity_pin.json").mkdir(parents=True)++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]+ assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output+ assert "To keep it on across restarts" in output+ assert "pin could not be recorded" in output+++def test_verbosity_apply_does_not_record_a_pin_without_a_process_identity(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ """No pid means the pin cannot be attributed later, so leave no record."""+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}})++ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)++ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]+ assert not (tmp_path / "ws" / "verbosity_pin.json").exists()+ assert "pin could not be recorded" in output+++def test_verbosity_apply_preserves_a_changed_raw_pin_with_the_same_clamped_level(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ proxy = _FakeProxy(+ {"mode": "cache", "pid": 4242, "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "8"}}+ )+ output = _apply_learned_level(+ monkeypatch,+ runner,+ tmp_path,+ proxy,+ level=3,+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 4},+ )+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]+ assert "level 4" in output+++ "payload",+ [+ {"status": "healthy"}, # no config block (e.g. a non-loopback caller's view)+ {"config": "unexpected"},+ ],+ ids=["no-config", "config-not-an-object"],+)+def test_query_proxy_verbosity_without_a_config_block(+ monkeypatch: pytest.MonkeyPatch, payload: dict+) -> None:+ import io++ from headroom.cli.learn import _query_proxy_verbosity++ monkeypatch.setattr(+ "urllib.request.urlopen", lambda *a, **k: io.BytesIO(json.dumps(payload).encode())+ )++ assert _query_proxy_verbosity(_PROXY_PORT) == (None, None, None)+++ ("overrides", "expected"),+ [+ ({}, (4242, _STARTED_AT, None)),+ ({"uptime_seconds": 0.0}, None), # the proxy has not recorded its start+ ({"uptime_seconds": None}, None),+ ({"timestamp": "not a time"}, None),+ ({"timestamp": None}, None),+ ],+ ids=["identified", "zero-uptime", "no-uptime", "bad-timestamp", "no-timestamp"],+)+def test_proxy_process_needs_a_start_time(overrides: dict, expected: tuple | None) -> None:+ from datetime import datetime++ from headroom.cli.learn import _proxy_process++ now = datetime.fromisoformat(_HEALTH_NOW.replace("Z", "+00:00")).timestamp()+ payload = {"timestamp": _HEALTH_NOW, "uptime_seconds": now - _STARTED_AT, **overrides}++ assert _proxy_process(payload, {"pid": 4242}) == expecteddiff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.pyindex 4b221a110..ff9a71a07 100644--- a/tests/test_offline_egress_chokepoint.py+++ b/tests/test_offline_egress_chokepoint.py@@ -2314,8 +2314,9 @@ _EGRESS_ALLOWLIST: dict[str, tuple[int, str]] = {"the locally running proxy (/health and /admin/runtime-env).",),"cli/learn.py": (- 1,- "loopback: hard-coded http://127.0.0.1:<port>/admin/runtime-env on the local proxy.",+ 2,+ "loopback: both sites are hard-coded http://127.0.0.1:<port> calls to "+ "the locally running proxy (/health and /admin/runtime-env).",),"cli/mcp.py": (1,diff --git a/wiki/learn.md b/wiki/learn.mdindex 657dae18d..3a180f6d8 100644--- a/wiki/learn.md+++ b/wiki/learn.md@@ -169,12 +169,22 @@ Options:`headroom learn --verbosity` analyzes past sessions to infer the ideal output verbosity level for your project and writes a `verbosity.json` profile.-**Important**: the output shaper is **off by default** and requires the `beta`-runtime rollout channel. Running `--verbosity --apply` will either:+**Important**: the output shaper is **off by default** (`HEADROOM_OUTPUT_SHAPER=1`+turns it on; it is available on every rollout channel). Running `--verbosity --apply` will either:- Hot-enable the output shaper on an eligible running proxy (`POST /admin/runtime-env`), OR-- Print instructions to set `HEADROOM_ROLLOUT_CHANNEL=beta` and `HEADROOM_OUTPUT_SHAPER=1` before `headroom wrap ...`--To keep the shaper on across proxy restarts, export both variables before starting the proxy.+- Print instructions to set `HEADROOM_OUTPUT_SHAPER=1` before `headroom wrap ...`++A proxy in cache mode (the default) does not read `verbosity.json`: a level that+changes mid-conversation would bust the prompt cache, so it steers at its startup+level. On a cache-mode proxy `--apply` therefore also pins+`HEADROOM_VERBOSITY_LEVEL` to the learned level, and records the pin (port,+proxy pid, level) in `verbosity_pin.json`. A `HEADROOM_VERBOSITY_LEVEL` that is+already set is left alone, and `--apply` says so. The one exception is a pin that+record proves an earlier `--apply` set: the same proxy process still holding the+same raw value. That pin is replaced. A changed raw pin remains operator-owned+even if it clamps to the same effective level as the previous CLI pin.++To keep the shaper on across proxy restarts, export `HEADROOM_OUTPUT_SHAPER=1` before starting the proxy, plus `HEADROOM_VERBOSITY_LEVEL=<level>` in cache mode.**Flag interactions**:- `--all` and `--project` are mutually exclusive