Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(learn): pin the learned verbosity level on a cache-mode proxy (#4033)

Kenneth Wong71663b44e26f
diff --git a/README.md b/README.md
index c9561ff9d..a478cf7f8 100644
--- a/README.md
+++ b/README.md
@@ -211,9 +211,14 @@ picks the level:
```bash
headroom learn --verbosity # dry run — preview what it found
-headroom learn --verbosity --apply # save it; the proxy picks it up
+headroom learn --verbosity --apply # save it and apply it to the running proxy
```
+A proxy in cache mode (the default) never re-reads the learned level, because a
+level that changes mid-conversation would bust the prompt cache. There `--apply`
+pins it with `HEADROOM_VERBOSITY_LEVEL`; set that variable yourself to keep it
+after a restart.
+
**Measuring it.** Output savings are counterfactual — we never see what the model
*would* have written — so Headroom reports an estimate with a confidence range
and labels it as one:
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index da1303816..47269c11f 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, NamedTuple
import click
@@ -418,7 +418,167 @@ def _make_llm_judge(model: str) -> Any:
return judge
-def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:
+def _local_proxy_port(port: int | None = None) -> int:
+ import os as _os
+
+ return port if port is not None else int(_os.environ.get("HEADROOM_PORT", "8787"))
+
+
+def _steered_level(raw: object) -> int | None:
+ """The level a proxy steers at for a raw ``HEADROOM_VERBOSITY_LEVEL``.
+
+ Mirrors ``OutputShaperSettings.from_env``: unparseable text falls back to
+ the default level, anything else is clamped to 0-4. ``None`` when unset.
+ """
+ from ..proxy.output_shaper import DEFAULT_VERBOSITY_LEVEL
+
+ if raw is None or str(raw).strip() == "":
+ return None
+ try:
+ level = int(str(raw).strip())
+ except ValueError:
+ level = DEFAULT_VERBOSITY_LEVEL
+ return max(0, min(4, level))
+
+
+class _ProxyProcess(NamedTuple):
+ """Process identity and its raw verbosity pin from the same health snapshot."""
+
+ pid: int
+ started_at: float
+ verbosity_pin: str | None = None
+
+
+def _proxy_process(payload: dict[str, Any], config: dict[str, Any]) -> _ProxyProcess | None:
+ """The proxy process behind a ``/health`` payload, or ``None`` if unknown.
+
+ The start time is ``timestamp - uptime_seconds``: the proxy computes both
+ from ``time.time()`` while building the payload, so the difference is its
+ start time to the millisecond, the same on every call.
+ """
+ from datetime import datetime
+
+ pid = config.get("pid")
+ timestamp = payload.get("timestamp")
+ uptime = payload.get("uptime_seconds")
+ if not isinstance(pid, int) or not isinstance(timestamp, str):
+ return None
+ # A proxy that has not recorded its start reports 0.0; that is not an identity.
+ if not isinstance(uptime, int | float) or uptime <= 0:
+ return None
+ try:
+ now = datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp()
+ except ValueError:
+ return None
+ runtime = config.get("runtime_env")
+ raw_pin = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None
+ return _ProxyProcess(
+ pid, round(now - float(uptime), 3), raw_pin if isinstance(raw_pin, str) else None
+ )
+
+
+def _query_proxy_verbosity(
+ port: int | None = None,
+) -> tuple[str | None, int | None, _ProxyProcess | None]:
+ """Best-effort: read a running local proxy's mode, pinned level and process.
+
+ All three come from ``/health``, whose config block the proxy includes for
+ loopback callers. Returns ``(mode, level, process)``, each ``None`` when the
+ proxy is unreachable or does not report it; ``level`` is the level its
+ ``HEADROOM_VERBOSITY_LEVEL`` pins, ``None`` when that is unset.
+ """
+ import json as _json
+ import urllib.error
+ import urllib.request
+
+ url = f"http://127.0.0.1:{_local_proxy_port(port)}/health"
+ try:
+ with urllib.request.urlopen(url, timeout=2) as response:
+ payload = _json.loads(response.read() or b"{}")
+ except (urllib.error.URLError, OSError, ValueError):
+ return None, None, None
+ config = payload.get("config") if isinstance(payload, dict) else None
+ if not isinstance(config, dict):
+ return None, None, None
+ mode = config.get("mode")
+ runtime = config.get("runtime_env")
+ level = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None
+ return (
+ mode if isinstance(mode, str) else None,
+ _steered_level(level),
+ _proxy_process(payload, config),
+ )
+
+
+#: Records the HEADROOM_VERBOSITY_LEVEL that ``--apply`` hot-pinned, and on
+#: which proxy process. Hot overrides die with the process, so a pin is ours to
+#: replace only while that same process still holds that value.
+_VERBOSITY_PIN_FILE = "verbosity_pin.json"
+
+#: Slack when comparing recorded and current start times, which agree to the
+#: millisecond for one process; a different process started this close to the
+#: old one would also need its pid, port and level to match.
+_START_TIME_TOLERANCE_S = 1.0
+
+
+def _pin_is_ours(ws: Path, port: int, process: _ProxyProcess | None, level: int | None) -> bool:
+ import json as _json
+
+ if process is None or level is None:
+ return False
+ try:
+ record = _json.loads((ws / _VERBOSITY_PIN_FILE).read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return False
+ if not isinstance(record, dict):
+ return False
+ started_at = record.get("started_at")
+ return (
+ record.get("port") == port
+ and record.get("pid") == process.pid
+ and record.get("level") == level
+ and process.verbosity_pin == str(level)
+ and isinstance(started_at, int | float)
+ and abs(started_at - process.started_at) <= _START_TIME_TOLERANCE_S
+ )
+
+
+def _record_pin(ws: Path, port: int, process: _ProxyProcess | None, level: int) -> bool:
+ """Record a pin ``--apply`` set. Returns False when it could not be recorded.
+
+ Best-effort: the pin is already live on the proxy, so a failed record only
+ means a later ``--apply`` treats it as the operator's and leaves it alone.
+ """
+ import json as _json
+
+ path = ws / _VERBOSITY_PIN_FILE
+ try:
+ if process is None:
+ # Without a process identity the pin cannot be attributed later.
+ path.unlink(missing_ok=True)
+ return False
+ record = {
+ "port": port,
+ "pid": process.pid,
+ "started_at": process.started_at,
+ "level": level,
+ }
+ path.write_text(_json.dumps(record), encoding="utf-8", newline="\n")
+ except OSError:
+ return False
+ return True
+
+
+class _ShaperActivation(NamedTuple):
+ status: str
+ port: int
+ #: The overrides the proxy reports it applied (empty if it did not answer).
+ applied: dict[str, str]
+
+
+def _activate_output_shaper(
+ port: int | None = None, *, verbosity_level: str | None = None
+) -> _ShaperActivation:
"""Best-effort: turn the output shaper ON for a running local proxy.
Writing ``verbosity.json`` is inert on its own — the shaper is a live,
@@ -426,20 +586,24 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:
``HEADROOM_OUTPUT_SHAPER`` is enabled in the proxy that serves traffic.
When a proxy is already running locally we hot-enable it via
``/admin/runtime-env`` (no restart, the same channel ``wrap`` uses), so
- ``--apply`` actually takes effect. Returns ``(status, port)`` where status is
- ``"live"`` (enabled on a running proxy), ``"blocked"`` (the proxy's
- rollout channel rejected it), ``"absent"`` (no reachable proxy), or
+ ``--apply`` actually takes effect. ``verbosity_level`` also pins
+ ``HEADROOM_VERBOSITY_LEVEL`` in the same request. Returns ``(status, port,
+ applied)`` where status is ``"live"`` (enabled on a running proxy),
+ ``"blocked"`` (the proxy's rollout config disables the shaper; the proxy
+ still applies the overrides), ``"absent"`` (no reachable proxy), or
``"error"``.
"""
import json as _json
- import os as _os
import urllib.error
import urllib.request
- resolved_port = port if port is not None else int(_os.environ.get("HEADROOM_PORT", "8787"))
+ resolved_port = _local_proxy_port(port)
+ overrides = {"HEADROOM_OUTPUT_SHAPER": "1"}
+ if verbosity_level is not None:
+ overrides["HEADROOM_VERBOSITY_LEVEL"] = verbosity_level
request = urllib.request.Request(
f"http://127.0.0.1:{resolved_port}/admin/runtime-env",
- data=_json.dumps({"HEADROOM_OUTPUT_SHAPER": "1"}).encode("utf-8"),
+ data=_json.dumps(overrides).encode("utf-8"),
method="POST",
headers={"Content-Type": "application/json"},
)
@@ -447,6 +611,12 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:
with urllib.request.urlopen(request, timeout=2) as response:
raw_response = response.read()
payload = _json.loads(raw_response) if raw_response else {}
+ applied_raw = payload.get("applied") if isinstance(payload, dict) else None
+ applied = (
+ {k: v for k, v in applied_raw.items() if isinstance(v, str)}
+ if isinstance(applied_raw, dict)
+ else {}
+ )
rollout = payload.get("rollout") if isinstance(payload, dict) else None
if isinstance(rollout, dict):
decisions = rollout.get("features")
@@ -460,13 +630,13 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]:
None,
)
if isinstance(output_shaper, dict) and not output_shaper.get("enabled", False):
- return "blocked", resolved_port
- return "live", resolved_port
+ return _ShaperActivation("blocked", resolved_port, applied)
+ return _ShaperActivation("live", resolved_port, applied)
except (urllib.error.URLError, OSError):
# ConnectionRefused (no proxy) or 404 (proxy predates the endpoint).
- return "absent", resolved_port
+ return _ShaperActivation("absent", resolved_port, {})
except ValueError:
- return "error", resolved_port
+ return _ShaperActivation("error", resolved_port, {})
def _run_verbosity(
@@ -593,24 +763,79 @@ def _run_verbosity(
# Writing the level is not enough — the shaper is off by default.
# Make --apply actually take effect: hot-enable a running proxy, and
# otherwise tell the user exactly how to turn it on.
- status, shaper_port = _activate_output_shaper()
- if status == "live":
+ level = best_profile.level
+ port = _local_proxy_port()
+ mode, pinned, process = _query_proxy_verbosity(port)
+ # Cache mode never reads verbosity.json: a level that can change while
+ # conversations are open would bust their prefix cache, so the proxy
+ # steers at its startup level unless HEADROOM_VERBOSITY_LEVEL pins one
+ # (resolve_verbosity_level). Pin the learned level there; otherwise the
+ # proxy would keep steering at its default while we report this level.
+ # An existing pin is the operator's and is left alone, unless the record
+ # from an earlier --apply shows this same proxy process holds the value
+ # that run pinned.
+ ours = _pin_is_ours(ws, port, process, pinned)
+ pin = str(level) if mode == "cache" and (pinned is None or ours) else None
+ status, shaper_port, applied = _activate_output_shaper(port, verbosity_level=pin)
+ # Record the pin whenever the proxy applied it: it stores overrides even
+ # when its rollout config disables the shaper ("blocked").
+ pin_applied = pin is not None and applied.get("HEADROOM_VERBOSITY_LEVEL") == pin
+ pin_recorded = pin_applied and _record_pin(ws, shaper_port, process, level)
+ cache_mode_hint = (
+ f"in cache mode (the default) also set HEADROOM_VERBOSITY_LEVEL={level}, "
+ "because cache mode does not read the learned level"
+ )
+ if status == "live" and pinned is not None and pin is None and pinned != level:
+ click.echo(
+ f"\n ⚠ Output shaper enabled on the running proxy (port {shaper_port}), "
+ f"but it stays at level {pinned}: HEADROOM_VERBOSITY_LEVEL is set there "
+ "and outranks the learned level."
+ )
+ click.echo(
+ f" To use level {level}, restart the proxy with HEADROOM_VERBOSITY_LEVEL={level}."
+ )
+ elif status == "live" and mode is None:
+ click.echo(
+ f"\n ⚠ Output shaper enabled on the running proxy (port {shaper_port}), but "
+ f"its mode could not be read, so level {level} may not be in use."
+ )
+ click.echo(
+ f" A proxy in cache mode (the default) keeps its startup level; restart it "
+ f"with HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL={level} to be sure."
+ )
+ elif status == "live":
+ how = (
+ " (pinned with HEADROOM_VERBOSITY_LEVEL: cache mode does not read the "
+ "learned level)"
+ if pin
+ else ""
+ )
click.echo(
f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "
- f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)."
+ f"level {level} is live now{how}."
+ )
+ if pin and pinned != level:
+ click.echo(
+ " Conversations already open re-cache their prompt once, because "
+ "the steering text changed."
+ )
+ restart_env = "HEADROOM_OUTPUT_SHAPER=1" + (
+ f" HEADROOM_VERBOSITY_LEVEL={level}" if mode == "cache" else ""
)
click.echo(
- " To keep it on across restarts: export HEADROOM_ROLLOUT_CHANNEL=beta "
- "and HEADROOM_OUTPUT_SHAPER=1 before `headroom wrap ...`."
+ f" To keep it on across restarts: export {restart_env} "
+ "before `headroom wrap ...` or `headroom proxy`."
)
elif status == "blocked":
+ # proxy_output_shaper is available on every channel, so only an
+ # explicit disable (HEADROOM_DISABLE_FEATURES) refuses the hot enable.
click.echo(
- "\n ⚠ Level written, but the running proxy's rollout channel blocks the "
- "beta output shaper."
+ "\n ⚠ Level written, but the running proxy's rollout config disables the "
+ "output shaper."
)
click.echo(
- " Restart it with HEADROOM_ROLLOUT_CHANNEL=beta and "
- "HEADROOM_OUTPUT_SHAPER=1; the learned level will be used automatically."
+ " Restart it without proxy_output_shaper in HEADROOM_DISABLE_FEATURES "
+ f"and with HEADROOM_OUTPUT_SHAPER=1; {cache_mode_hint}."
)
else:
click.echo(
@@ -618,10 +843,15 @@ def _run_verbosity(
"NOT shaping output yet."
)
click.echo(
- " Enable it: export HEADROOM_ROLLOUT_CHANNEL=beta and "
- "HEADROOM_OUTPUT_SHAPER=1, then run `headroom wrap ...` (or restart "
- "`headroom proxy`). The learned level is then used automatically while "
- "HEADROOM_VERBOSITY_LEVEL is unset."
+ " Enable it: export HEADROOM_OUTPUT_SHAPER=1, then run `headroom wrap ...` "
+ "(or restart `headroom proxy`). A token-mode proxy then uses the learned "
+ f"level; {cache_mode_hint}."
+ )
+ if pin_applied and not pin_recorded:
+ click.echo(
+ f" Note: the HEADROOM_VERBOSITY_LEVEL={level} pin could not be recorded in "
+ f"{ws / _VERBOSITY_PIN_FILE}, so a later --apply will leave it in place as "
+ "if you had set it."
)
else:
click.echo("\n Dry run — use --apply to persist the level and baseline.")
diff --git a/tests/test_cli_learn.py b/tests/test_cli_learn.py
index 8ebc7e858..95a52d078 100644
--- a/tests/test_cli_learn.py
+++ b/tests/test_cli_learn.py
@@ -330,6 +330,8 @@ def test_verbosity_all_apply_aggregates_baselines_across_projects(
monkeypatch.setattr("headroom.learn.registry.get_plugin", lambda name: plugin)
monkeypatch.setattr("headroom.learn.verbosity.analyze", fake_analyze)
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))
+ # --apply talks to a local proxy; keep it off whatever is listening on 8787.
+ monkeypatch.setattr("urllib.request.urlopen", _no_local_proxy)
result = runner.invoke(
main,
@@ -344,6 +346,9 @@ def test_verbosity_all_apply_aggregates_baselines_across_projects(
assert "opus|new_user_ask|s|tools" in ledger.baseline.strata
assert "sonnet|unknown|m|notools" in ledger.baseline.strata
assert "across 2 project(s)" in result.output
+ # The shaper is available on every channel; the hint must not ask for beta.
+ assert "HEADROOM_OUTPUT_SHAPER=1" in result.output
+ assert "HEADROOM_ROLLOUT_CHANNEL" not in result.output
# The applied level comes from the project with the most samples (A → 1).
verbosity = _json.loads((tmp_path / "ws" / "verbosity.json").read_text())
assert verbosity["level"] == 1
@@ -637,7 +642,7 @@ def test_activate_output_shaper_reports_effective_rollout_decision(
monkeypatch.setattr(urllib.request, "urlopen", lambda *args, **kwargs: Response())
- status, port = _activate_output_shaper(9876)
+ status, port, _ = _activate_output_shaper(9876)
assert status == expected
assert port == 9876
@@ -662,4 +667,432 @@ def test_activate_output_shaper_handles_malformed_response(
monkeypatch.setattr(urllib.request, "urlopen", lambda *args, **kwargs: Response())
- assert _activate_output_shaper(9876) == ("error", 9876)
+ assert _activate_output_shaper(9876) == ("error", 9876, {})
+
+
+def _no_local_proxy(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ import urllib.error
+
+ raise urllib.error.URLError("no proxy in tests")
+
+
+# Where the stubbed local proxy listens; HEADROOM_PORT points the CLI at it.
+_PROXY_PORT = 18787
+# The fake proxy's start time and the wall clock it answers /health at.
+_STARTED_AT = 1_791_400_000.0
+_HEALTH_NOW = "2026-10-08T10:00:00Z"
+
+
+class _FakeProxy:
+ """A local proxy as ``urllib.request.urlopen`` sees it.
+
+ ``health_config`` is the ``/health`` config block (``None`` = unreachable);
+ POSTs to ``/admin/runtime-env`` are recorded and accepted.
+ """
+
+ def __init__(
+ self,
+ health_config: dict | None,
+ *,
+ shaper_allowed: bool = True,
+ started_at: float = _STARTED_AT,
+ ) -> None:
+ self.health_config = health_config
+ self.shaper_allowed = shaper_allowed
+ self.started_at = started_at
+ self.posted: list[dict] = []
+
+ def urlopen(self, request, timeout=None): # noqa: ANN001, ANN201
+ import io
+ import urllib.error
+
+ url = request if isinstance(request, str) else request.full_url
+ if url.endswith("/health"):
+ if self.health_config is None:
+ raise urllib.error.URLError("unreachable")
+ from datetime import datetime
+
+ now = datetime.fromisoformat(_HEALTH_NOW.replace("Z", "+00:00")).timestamp()
+ payload = {
+ "timestamp": _HEALTH_NOW,
+ "uptime_seconds": round(now - self.started_at, 3),
+ "config": self.health_config,
+ }
+ return io.BytesIO(json.dumps(payload).encode())
+ self.posted.append(json.loads(request.data))
+ rollout = {"features": [{"name": "proxy_output_shaper", "enabled": self.shaper_allowed}]}
+ return io.BytesIO(json.dumps({"applied": self.posted[-1], "rollout": rollout}).encode())
+
+
+def _apply_learned_level(
+ monkeypatch: pytest.MonkeyPatch,
+ runner: CliRunner,
+ tmp_path: Path,
+ proxy: _FakeProxy,
+ *,
+ level: int = 3,
+ pin_record: dict | list | None = None,
+ previous_level: int | None = None,
+) -> str:
+ """Run ``learn --verbosity --apply`` for one project that learns ``level``.
+
+ ``pin_record`` seeds the record an earlier ``--apply`` leaves after pinning,
+ ``previous_level`` the verbosity.json it saved. The proxy listens on
+ ``_PROXY_PORT``.
+ """
+ from headroom.proxy.output_savings import BaselineModel
+
+ workspace = tmp_path / "ws"
+ workspace.mkdir(exist_ok=True)
+ if pin_record is not None:
+ (workspace / "verbosity_pin.json").write_text(json.dumps(pin_record), newline="\n")
+ if previous_level is not None:
+ (workspace / "verbosity.json").write_text(
+ json.dumps({"verbosity_level": previous_level}), newline="\n"
+ )
+ data_dir = tmp_path / "sessions"
+ data_dir.mkdir()
+ (data_dir / "s.jsonl").write_text("{}")
+ project = SimpleNamespace(
+ name="p", project_path=tmp_path / "src", data_path=data_dir, extra_data_paths=[]
+ )
+ baseline = BaselineModel()
+ baseline.observe("opus|new_user_ask|s|tools", 100)
+ profile = SimpleNamespace(
+ level=level,
+ confidence="high",
+ source="heuristic",
+ rationale="test",
+ signals={},
+ learned_at=None,
+ save=lambda path: Path(str(path)).write_text(
+ json.dumps({"verbosity_level": level}), newline="\n"
+ ),
+ )
+ monkeypatch.setattr(
+ "headroom.learn.registry.get_plugin", lambda name: FakePlugin(name, "Claude", [project])
+ )
+ monkeypatch.setattr(
+ "headroom.learn.verbosity.analyze",
+ lambda session_paths, project_path, llm_judge=None: (profile, baseline),
+ )
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(workspace))
+ monkeypatch.setenv("HEADROOM_PORT", str(_PROXY_PORT))
+ monkeypatch.setattr("urllib.request.urlopen", proxy.urlopen)
+
+ result = runner.invoke(
+ main,
+ ["learn", "--agent", "claude", "--verbosity", "--all", "--apply"],
+ catch_exceptions=False,
+ )
+ assert result.exit_code == 0, result.output
+ return result.output
+
+
+def test_verbosity_apply_pins_the_level_on_a_cache_mode_proxy(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """Cache mode never reads verbosity.json, so "live" needs the level pinned."""
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": None}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
+ assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
+ assert "HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL=3 before" in output
+ assert "re-cache their prompt once" in output
+
+
+def test_verbosity_apply_leaves_a_token_mode_proxy_reading_the_profile(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """Token mode reads verbosity.json per request; a pin would only shadow it."""
+ proxy = _FakeProxy({"mode": "token", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": None}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "level 3 is live now." in output
+ assert "export HEADROOM_OUTPUT_SHAPER=1 before" in output
+
+
+def test_verbosity_apply_does_not_override_an_explicit_level(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "2"}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "is live now" not in output
+ assert "it stays at level 2: HEADROOM_VERBOSITY_LEVEL is set there" in output
+
+
+def test_verbosity_apply_records_the_pin_it_sets(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})
+
+ _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3}
+
+
+def test_verbosity_apply_replaces_the_pin_an_earlier_apply_set(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """A re-learned level must not be shadowed by the previous run's own pin."""
+ proxy = _FakeProxy(
+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
+ )
+
+ output = _apply_learned_level(
+ monkeypatch,
+ runner,
+ tmp_path,
+ proxy,
+ level=1,
+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3},
+ )
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "1"}]
+ assert "level 1 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
+ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 1}
+
+
+def test_verbosity_apply_re_pinning_the_same_level_does_not_warn_about_re_caching(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy(
+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
+ )
+
+ output = _apply_learned_level(
+ monkeypatch,
+ runner,
+ tmp_path,
+ proxy,
+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3},
+ )
+
+ assert "level 3 is live now" in output
+ assert "re-cache" not in output
+
+
+ "pin_record",
+ [
+ None, # the operator pinned the level that was learned before
+ # pinned on a proxy since restarted
+ {"port": _PROXY_PORT, "pid": 1111, "started_at": _STARTED_AT, "level": 3},
+ # a new process that reused the old one's pid on the same port
+ {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT - 3600, "level": 3},
+ # the operator re-pinned after us
+ {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 2},
+ # a record from before start times were recorded
+ {"port": _PROXY_PORT, "pid": 4242, "level": 3},
+ # a record that is valid JSON but not an object
+ [_PROXY_PORT, 4242, _STARTED_AT, 3],
+ ],
+ ids=[
+ "no-record",
+ "restarted-proxy",
+ "reused-pid",
+ "changed-since",
+ "no-start-time",
+ "not-an-object",
+ ],
+)
+def test_verbosity_apply_keeps_a_pin_it_cannot_prove_it_set(
+ monkeypatch: pytest.MonkeyPatch,
+ runner: CliRunner,
+ tmp_path: Path,
+ pin_record: dict | list | None,
+) -> None:
+ proxy = _FakeProxy(
+ {"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
+ )
+
+ output = _apply_learned_level(
+ monkeypatch, runner, tmp_path, proxy, level=1, pin_record=pin_record, previous_level=3
+ )
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "it stays at level 3: HEADROOM_VERBOSITY_LEVEL is set there" in output
+
+
+def test_verbosity_pin_record_is_written_with_lf(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001
+ from headroom.cli import learn as learn_cli
+
+ calls: list[dict] = []
+ real_write_text = Path.write_text
+
+ def write_text(self, data, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
+ calls.append(kwargs)
+ return real_write_text(self, data, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "write_text", write_text)
+ process = learn_cli._ProxyProcess(4242, _STARTED_AT)
+ assert learn_cli._record_pin(tmp_path, _PROXY_PORT, process, 3) is True
+
+ assert [call.get("newline") for call in calls] == ["\n"]
+
+
[email protected]("raw", ["9", " 04 "])
+def test_verbosity_apply_reads_a_pin_the_way_the_proxy_clamps_it(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path, raw: str
+) -> None:
+ """The proxy steers a pin of 9 (or " 04 ") at level 4, so it matches a learned 4."""
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": raw}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=4)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "level 4 is live now." in output
+ assert "stays at level" not in output
+
+
+def test_verbosity_apply_reads_an_unparseable_pin_as_the_default_level(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """The proxy steers a non-numeric pin at the default level, and so must the CLI."""
+ from headroom.proxy.output_shaper import DEFAULT_VERBOSITY_LEVEL
+
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "high"}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=3)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert f"it stays at level {DEFAULT_VERBOSITY_LEVEL}: HEADROOM_VERBOSITY_LEVEL" in output
+
+
+def test_verbosity_apply_does_not_claim_live_when_the_mode_is_unknown(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy(None)
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "is live now" not in output
+ assert "its mode could not be read" in output
+
+
+def test_verbosity_apply_on_a_proxy_that_disables_the_shaper(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}}, shaper_allowed=False)
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert "rollout config disables the output shaper" in output
+ assert "HEADROOM_DISABLE_FEATURES" in output
+ assert "HEADROOM_VERBOSITY_LEVEL=3" in output
+ assert "HEADROOM_ROLLOUT_CHANNEL" not in output
+
+
+def test_verbosity_apply_records_a_pin_the_proxy_applied_while_the_shaper_is_disabled(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """The proxy stores the pin even when it reports the shaper blocked; own it."""
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242}, shaper_allowed=False)
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
+ assert "rollout config disables the output shaper" in output
+ record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
+ assert record == {"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 3}
+
+
+def test_verbosity_apply_reports_activation_when_the_pin_cannot_be_recorded(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})
+ # A directory where the record file should go makes the write fail.
+ (tmp_path / "ws" / "verbosity_pin.json").mkdir(parents=True)
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
+ assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
+ assert "To keep it on across restarts" in output
+ assert "pin could not be recorded" in output
+
+
+def test_verbosity_apply_does_not_record_a_pin_without_a_process_identity(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ """No pid means the pin cannot be attributed later, so leave no record."""
+ proxy = _FakeProxy({"mode": "cache", "runtime_env": {}})
+
+ output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy)
+
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
+ assert not (tmp_path / "ws" / "verbosity_pin.json").exists()
+ assert "pin could not be recorded" in output
+
+
+def test_verbosity_apply_preserves_a_changed_raw_pin_with_the_same_clamped_level(
+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
+) -> None:
+ proxy = _FakeProxy(
+ {"mode": "cache", "pid": 4242, "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "8"}}
+ )
+ output = _apply_learned_level(
+ monkeypatch,
+ runner,
+ tmp_path,
+ proxy,
+ level=3,
+ pin_record={"port": _PROXY_PORT, "pid": 4242, "started_at": _STARTED_AT, "level": 4},
+ )
+ assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
+ assert "level 4" in output
+
+
+ "payload",
+ [
+ {"status": "healthy"}, # no config block (e.g. a non-loopback caller's view)
+ {"config": "unexpected"},
+ ],
+ ids=["no-config", "config-not-an-object"],
+)
+def test_query_proxy_verbosity_without_a_config_block(
+ monkeypatch: pytest.MonkeyPatch, payload: dict
+) -> None:
+ import io
+
+ from headroom.cli.learn import _query_proxy_verbosity
+
+ monkeypatch.setattr(
+ "urllib.request.urlopen", lambda *a, **k: io.BytesIO(json.dumps(payload).encode())
+ )
+
+ assert _query_proxy_verbosity(_PROXY_PORT) == (None, None, None)
+
+
+ ("overrides", "expected"),
+ [
+ ({}, (4242, _STARTED_AT, None)),
+ ({"uptime_seconds": 0.0}, None), # the proxy has not recorded its start
+ ({"uptime_seconds": None}, None),
+ ({"timestamp": "not a time"}, None),
+ ({"timestamp": None}, None),
+ ],
+ ids=["identified", "zero-uptime", "no-uptime", "bad-timestamp", "no-timestamp"],
+)
+def test_proxy_process_needs_a_start_time(overrides: dict, expected: tuple | None) -> None:
+ from datetime import datetime
+
+ from headroom.cli.learn import _proxy_process
+
+ now = datetime.fromisoformat(_HEALTH_NOW.replace("Z", "+00:00")).timestamp()
+ payload = {"timestamp": _HEALTH_NOW, "uptime_seconds": now - _STARTED_AT, **overrides}
+
+ assert _proxy_process(payload, {"pid": 4242}) == expected
diff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.py
index 4b221a110..ff9a71a07 100644
--- a/tests/test_offline_egress_chokepoint.py
+++ b/tests/test_offline_egress_chokepoint.py
@@ -2314,8 +2314,9 @@ _EGRESS_ALLOWLIST: dict[str, tuple[int, str]] = {
"the locally running proxy (/health and /admin/runtime-env).",
),
"cli/learn.py": (
- 1,
- "loopback: hard-coded http://127.0.0.1:<port>/admin/runtime-env on the local proxy.",
+ 2,
+ "loopback: both sites are hard-coded http://127.0.0.1:<port> calls to "
+ "the locally running proxy (/health and /admin/runtime-env).",
),
"cli/mcp.py": (
1,
diff --git a/wiki/learn.md b/wiki/learn.md
index 657dae18d..3a180f6d8 100644
--- a/wiki/learn.md
+++ b/wiki/learn.md
@@ -169,12 +169,22 @@ Options:
`headroom learn --verbosity` analyzes past sessions to infer the ideal output verbosity level for your project and writes a `verbosity.json` profile.
-**Important**: the output shaper is **off by default** and requires the `beta`
-runtime rollout channel. Running `--verbosity --apply` will either:
+**Important**: the output shaper is **off by default** (`HEADROOM_OUTPUT_SHAPER=1`
+turns it on; it is available on every rollout channel). Running `--verbosity --apply` will either:
- Hot-enable the output shaper on an eligible running proxy (`POST /admin/runtime-env`), OR
-- Print instructions to set `HEADROOM_ROLLOUT_CHANNEL=beta` and `HEADROOM_OUTPUT_SHAPER=1` before `headroom wrap ...`
-
-To keep the shaper on across proxy restarts, export both variables before starting the proxy.
+- Print instructions to set `HEADROOM_OUTPUT_SHAPER=1` before `headroom wrap ...`
+
+A proxy in cache mode (the default) does not read `verbosity.json`: a level that
+changes mid-conversation would bust the prompt cache, so it steers at its startup
+level. On a cache-mode proxy `--apply` therefore also pins
+`HEADROOM_VERBOSITY_LEVEL` to the learned level, and records the pin (port,
+proxy pid, level) in `verbosity_pin.json`. A `HEADROOM_VERBOSITY_LEVEL` that is
+already set is left alone, and `--apply` says so. The one exception is a pin that
+record proves an earlier `--apply` set: the same proxy process still holding the
+same raw value. That pin is replaced. A changed raw pin remains operator-owned
+even if it clamps to the same effective level as the previous CLI pin.
+
+To keep the shaper on across proxy restarts, export `HEADROOM_OUTPUT_SHAPER=1` before starting the proxy, plus `HEADROOM_VERBOSITY_LEVEL=<level>` in cache mode.
**Flag interactions**:
- `--all` and `--project` are mutually exclusive