Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): make the output shaper actually reduce output in the default mode (#3773)
Tejas Choprac02929846caf
diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.pyindex c0989a8d9..39e6f6c45 100644--- a/headroom/proxy/output_shaper.py+++ b/headroom/proxy/output_shaper.py@@ -62,6 +62,7 @@ from headroom.rollout import FeatureDecisionReasonlogger = logging.getLogger(__name__)__all__ = [+ "DEFAULT_VERBOSITY_LEVEL","OutputShaperSettings","ShapeResult","TurnKind",@@ -80,12 +81,23 @@ __all__ = [_replace_or_append_steering_block = replace_or_append_steering_block+#: Level handed to an operator who turned the shaper on and said nothing else.+#: Two, not three: L3 instructs the model to drop content ("give conclusions+#: only; omit rationale"), which reads as the model getting worse to someone who+#: did not ask for it -- the same reasoning the feature's own rollout spec gives+#: for not defaulting the shaper on at all. L2 only forbids ceremony and+#: restatement. Measured on a long explain-with-code turn (3 paired runs,+#: non-overlapping ranges): 2375 -> 1663 output tokens, -30%, with no visible+#: loss of content.+DEFAULT_VERBOSITY_LEVEL = 2++@dataclass(frozen=True)class OutputShaperSettings:"""Output-shaping settings with rollout enablement injected by the proxy."""enabled: bool = False- verbosity_level: int = 3+ verbosity_level: int = DEFAULT_VERBOSITY_LEVEL# False in ``mode="cache"``. Steering is the one lever that writes into the# provider prefix-cache key (it appends to the system-prompt tail, and on a# body with no system field it creates one); effort routing and the@@ -110,9 +122,11 @@ class OutputShaperSettings:"yes",)try:- level = int(runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL", "3"))+ level = int(+ runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL", str(DEFAULT_VERBOSITY_LEVEL))+ )except ValueError:- level = 2+ level = DEFAULT_VERBOSITY_LEVELlevel = max(0, min(4, level))return cls(enabled=enabled,@@ -165,37 +179,41 @@ def steering_allowed_for(config: Any) -> bool:``mode="cache"`` freezes prior turns specifically to keep the provider'sprefix-cache key byte-stable (see ``ProxyConfig.mode``). Verbosity steering- writes into that key, so running it there trades a large, certain cache- cost for a small, uncertain output saving — the wrong side of a roughly- 60x margin on a long context. Effort routing and the thinking budget are- unaffected: they ride request parameters outside the cache key, so they- keep saving in cache mode.+ writes into that key, so a block that CHANGES there trades a large, certain+ cache cost for a small, uncertain output saving — the wrong side of a+ roughly 60x margin on a long context. Effort routing and the thinking+ budget are unaffected: they ride request parameters outside the cache key,+ so they keep saving in cache mode.++ This is the gate for the sources that can change mid-conversation. It is+ NOT the last word: :func:`resolve_verbosity_level` lets an explicitly+ pinned ``HEADROOM_VERBOSITY_LEVEL`` through even here, because a level+ fixed before startup is written into turn 1's prefix and never moves. See+ that function for the reasoning."""return getattr(config, "mode", None) != "cache"-def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:- """Resolve the live verbosity level and its source.+#: Keys already warned about. Resolution runs on every request of every+#: conversation, so an unguarded warning would be one log line per turn.+_REPORTED: set[str] = set()- Precedence:- 1. ``HEADROOM_VERBOSITY_LEVEL`` set explicitly → manual override.- 2. AIMD controller state (when ``HEADROOM_VERBOSITY_AUTOTUNE`` is on).- 3. Learned ``verbosity.json`` from ``learn --verbosity``.- 4. The settings default.- Returns ``(level, source)``. Kept separate from :func:`shape_request` so the- body-mutating core stays a pure function of an explicit level.- """- if not settings.steering_enabled:- # Level 0 is the documented "no steering" value, so this disables the- # only cache-key-mutating lever while leaving effort routing on. It- # deliberately outranks the manual override below: a level set in the- # environment must not reintroduce a prefix mutation the mode exists- # to prevent.- return 0, "cache_mode"- if runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL"):- return settings.verbosity_level, "env"+def _report_once(key: str, message: str, *args: Any) -> None:+ """Warn the first time only. Cleared by tests via ``_REPORTED.clear()``."""+ if key in _REPORTED:+ return+ _REPORTED.add(key)+ logger.warning(message, *args)++def _resolve_unpinned_level(settings: OutputShaperSettings) -> tuple[int, str]:+ """Resolve the level from the sources that can move under our feet.++ The controller rewrites its state file as it hunts, and ``verbosity.json``+ appears the moment someone runs ``learn --verbosity``. Both can therefore+ change mid-conversation, which is what makes them unsafe in cache mode.+ """try:from ..paths import workspace_dir@@ -231,6 +249,83 @@ def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:return settings.verbosity_level, "default"+def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:+ """Resolve the live verbosity level and its source.++ Precedence:+ 1. ``HEADROOM_VERBOSITY_LEVEL`` set explicitly → manual override.+ 2. AIMD controller state (when ``HEADROOM_VERBOSITY_AUTOTUNE`` is on).+ 3. Learned ``verbosity.json`` from ``learn --verbosity``.+ 4. The settings default.++ Returns ``(level, source)``. Kept separate from :func:`shape_request` so the+ body-mutating core stays a pure function of an explicit level.++ Cache mode and the pinned level+ -------------------------------+ ``mode="cache"`` used to force this to ``0`` unconditionally, outranking+ even an explicit ``HEADROOM_VERBOSITY_LEVEL``. That guarded the right+ hazard with the wrong instrument.++ What costs a prefix cache is the steering block *changing* — appearing on a+ conversation already in flight, or moving between levels — because the+ block sits in the ``system`` array, which is inside the cached prefix of+ every message-level ``cache_control`` breakpoint the client sets. What does+ NOT cost anything is the block simply *being there*: :func:`shape_request`+ applies it on every request with no turn-kind gating, and the text is+ byte-stable per level and idempotent via the sentinel. So a level pinned in+ the environment before the proxy starts is written into turn 1's prefix and+ is byte-identical on every turn after it. The cache is established+ including the block and hits normally from then on; the only cost is the+ block's own tokens riding along in the cached prefix (~80 at L2, about+ $0.000024 per turn at Sonnet cache-read rates, against a measured ~700+ output tokens saved per turn — roughly 450x the other way).++ What matters is therefore not WHICH level, but whether it can move. Both+ sources that cannot move are allowed through in cache mode:++ * an explicit ``HEADROOM_VERBOSITY_LEVEL`` → ``env_pinned``+ * otherwise the settings level, fixed at startup → ``cache_mode_default``++ The second is what makes ``HEADROOM_OUTPUT_SHAPER=1`` sufficient on its+ own. The shaper is opt-in (see the ``proxy_output_shaper`` rollout spec, which+ deliberately does not default-enable it), so an enabled shaper is already an+ explicit request and there is nothing further to ask the operator for.++ The controller and the learned profile stay out of cache mode entirely —+ those are exactly the sources that rewrite themselves while conversations+ are open. Skipping them also spares the default mode two filesystem stats+ per request.+ """+ if runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL"):+ return settings.verbosity_level, ("env" if settings.steering_enabled else "env_pinned")++ if not settings.steering_enabled:+ # Cache mode. The shaper is opt-in (see the ``proxy_output_shaper``+ # rollout spec), so an enabled shaper is already an explicit operator+ # act -- there is nothing further to ask for. Steer at the settings+ # level, which was fixed at startup and cannot move.+ #+ # The controller and the learned profile are deliberately NOT consulted+ # here. Both rewrite their files while conversations are open, and a+ # level that moves mid-conversation rewrites the system array, which+ # sits inside the cached prefix of every message-level cache_control+ # breakpoint. That is the bust this mode exists to prevent -- and+ # skipping the lookup also spares the default mode two filesystem stats+ # per request.+ if runtime_env.getenv("HEADROOM_VERBOSITY_AUTOTUNE", "").lower() in ("1", "true", "yes"):+ _report_once(+ "autotune_in_cache_mode",+ "OutputShaper: HEADROOM_VERBOSITY_AUTOTUNE is set but the AIMD controller is "+ "not consulted in cache mode — a level that moves mid-conversation busts the "+ "prefix cache. Steering at L%d. Set HEADROOM_MODE=token to autotune.",+ settings.verbosity_level,+ )+ return settings.verbosity_level, "cache_mode_default"++ return _resolve_unpinned_level(settings)++@dataclassclass ShapeResult:"""What the shaper did to a request body."""diff --git a/tests/gateway/test_output_steering.py b/tests/gateway/test_output_steering.pyindex 852a66365..565f90081 100644--- a/tests/gateway/test_output_steering.py+++ b/tests/gateway/test_output_steering.py@@ -209,16 +209,22 @@ def test_env_off_wins(make_headroom_client, monkeypatch) -> None:assert data["body"]["system"] == SYSTEM_PROMPT-def test_cache_mode_labels_but_never_steers(make_headroom_client, monkeypatch) -> None:- """``steering_allowed_for`` is false in ``mode="cache"``: the measurement- label still rides the outcome but the prefix-cache key is left alone."""+def test_cache_mode_steers_at_the_default_level(make_headroom_client, monkeypatch) -> None:+ """``HEADROOM_OUTPUT_SHAPER=1`` alone must steer, in the default mode.++ Cache mode used to resolve the level to 0 here, so the label rode the+ outcome while the body went out untouched -- the feature did nothing for+ the mode almost every deployment runs. A level fixed at startup cannot+ move mid-conversation, so it cannot bust the prefix cache, so it steers.+ """monkeypatch.setenv("HEADROOM_OUTPUT_SHAPER", "1")+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)client = make_headroom_client(mode="cache")for sent in (_anthropic_body(gateway={}), _openai_body(gateway={})):data = compress(client, sent).json()assert _stratum_label(data).startswith(STRATUM_PREFIX)- assert not any(t.startswith(VERBOSITY_PREFIX) for t in data["transforms_applied"])- assert STEERING_SENTINEL not in canonical(data["body"])+ assert any(t.startswith(VERBOSITY_PREFIX) for t in data["transforms_applied"])+ assert STEERING_SENTINEL in canonical(data["body"])def test_control_arm_labels_but_never_steers(make_headroom_client, monkeypatch) -> None:diff --git a/tests/test_output_shaper.py b/tests/test_output_shaper.pyindex 0c08d68f9..a609db09c 100644--- a/tests/test_output_shaper.py+++ b/tests/test_output_shaper.py@@ -10,6 +10,7 @@ import copyfrom typing import Anyfrom headroom.proxy.output_shaper import (+ DEFAULT_VERBOSITY_LEVEL,OutputShaperSettings,TurnKind,apply_openai_responses_verbosity_steering,@@ -180,10 +181,10 @@ class TestShapeRequest:}result = shape_request(body, ENABLED)assert result.changed is True- assert result.labels == ["output_shaper:verbosity:L3"]+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]assert body["output_config"]["effort"] == "xhigh", "must not touch effort"assert body["thinking"] == {"type": "adaptive"}, "must not touch thinking"- assert body["system"][1]["text"] == steering_text(3)+ assert body["system"][1]["text"] == steering_text(DEFAULT_VERBOSITY_LEVEL)def test_new_ask_gets_steering_but_keeps_effort(self):body = {@@ -192,7 +193,7 @@ class TestShapeRequest:"output_config": {"effort": "xhigh"},}result = shape_request(body, ENABLED)- assert result.labels == ["output_shaper:verbosity:L3"]+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]assert body["output_config"]["effort"] == "xhigh"def test_second_pass_is_stable(self):@@ -282,8 +283,8 @@ class TestShapeOpenAIChatRequest:}result = shape_openai_chat_request(body, ENABLED)assert result.changed is True- assert result.labels == ["output_shaper:verbosity:L3"]- assert steering_text(3) in body["messages"][0]["content"]+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]+ assert steering_text(DEFAULT_VERBOSITY_LEVEL) in body["messages"][0]["content"]# User turn is untouched.assert body["messages"][1] == {"role": "user", "content": "hi"}@@ -398,20 +399,144 @@ class TestCacheModeSuppressesSteeringOnly:assert steering_allowed_for(SimpleNamespace(mode="cache")) is Falseassert steering_allowed_for(None) is True, "absent config must not disable levers"- def test_cache_mode_resolves_level_zero(self):+ def test_cache_mode_steers_at_the_startup_level(self):+ """Cache mode used to force 0 here. What it must prevent is a level that+ MOVES; a level fixed at startup cannot, so it is steered at."""from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_levelsettings = OutputShaperSettings(enabled=True, verbosity_level=3, steering_enabled=False)- assert resolve_verbosity_level(settings) == (0, "cache_mode")+ assert resolve_verbosity_level(settings) == (3, "cache_mode_default")- def test_cache_mode_outranks_the_manual_level_override(self, monkeypatch):- """An env-set level must not reintroduce the prefix mutation."""+ def test_cache_mode_honours_a_pinned_manual_level(self, monkeypatch):+ """A level pinned before startup never moves, so it cannot bust a cache.++ The block lands in turn 1's prefix and is byte-identical on every turn+ after it, so the cached prefix is established WITH it and hits normally.+ Previously this resolved to 0 and the knob was silently ignored.+ """from headroom.proxy import runtime_envfrom headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_levelmonkeypatch.setattr(runtime_env, "getenv", lambda k, d="": "4" if "VERBOSITY" in k else d)settings = OutputShaperSettings(enabled=True, verbosity_level=4, steering_enabled=False)- assert resolve_verbosity_level(settings)[0] == 0+ assert resolve_verbosity_level(settings) == (4, "env_pinned")++ def test_shaper_alone_steers_at_l2_in_cache_mode(self, monkeypatch):+ """``HEADROOM_OUTPUT_SHAPER=1`` must be sufficient on its own.++ The shaper is opt-in, so an enabled shaper is already an explicit+ request; there is nothing further to ask the operator for. Previously+ this resolved to 0 and the feature did nothing in the default mode.+ """+ from headroom.proxy import runtime_env+ from headroom.proxy.output_shaper import (+ DEFAULT_VERBOSITY_LEVEL,+ OutputShaperSettings,+ resolve_verbosity_level,+ )++ monkeypatch.setattr(runtime_env, "getenv", lambda k, d="": d)+ settings = OutputShaperSettings.from_env(enabled=True, steering_enabled=False)+ assert settings.enabled is True+ assert resolve_verbosity_level(settings) == (DEFAULT_VERBOSITY_LEVEL, "cache_mode_default")+ assert DEFAULT_VERBOSITY_LEVEL == 2++ def test_cache_mode_ignores_a_learned_level(self, tmp_path, monkeypatch):+ """``verbosity.json`` appears the moment someone runs ``learn``, so it+ must not be consulted where a mid-conversation change busts a cache."""+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level++ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)+ (tmp_path / "verbosity.json").write_text('{"verbosity_level": 4}')++ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")++ def test_cache_mode_ignores_the_controller_and_says_so_once(+ self, tmp_path, monkeypatch, caplog+ ):+ """Autotune silently doing nothing is invisible from outside."""+ import logging++ from headroom.proxy import output_shaper+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level++ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))+ monkeypatch.setenv("HEADROOM_VERBOSITY_AUTOTUNE", "1")+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)+ (tmp_path / "verbosity_controller.json").write_text('{"level": 4}')+ output_shaper._REPORTED.clear()++ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)+ with caplog.at_level(logging.WARNING, logger="headroom.proxy.output_shaper"):+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")+ resolve_verbosity_level(settings)++ warnings = [r for r in caplog.records if "AUTOTUNE" in r.getMessage()]+ assert len(warnings) == 1, "must not reprint on every request"+ assert "HEADROOM_MODE=token" in warnings[0].getMessage()++ def test_cache_mode_reads_no_workspace_files(self, monkeypatch):+ """Resolution runs per request; the default mode must not stat files."""+ import headroom.paths as paths+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level++ def _boom():+ raise AssertionError("workspace_dir() must not be consulted in cache mode")++ monkeypatch.setattr(paths, "workspace_dir", _boom)+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")++ def test_pinned_level_keeps_the_system_array_byte_stable_across_turns(self, monkeypatch):+ """The cache-safety claim, asserted rather than argued.++ Ten turns of a growing conversation must produce a byte-identical+ ``system`` array -- that identity is the whole reason a pinned level+ costs no cache.+ """+ import json as _json++ from headroom.proxy import runtime_env+ from headroom.proxy.output_shaper import (+ OutputShaperSettings,+ resolve_verbosity_level,+ shape_request,+ )++ monkeypatch.setattr(runtime_env, "getenv", lambda k, d="": "2" if "VERBOSITY" in k else d)+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)+ level, source = resolve_verbosity_level(settings)+ assert (level, source) == (2, "env_pinned")++ systems = []+ messages = []+ for turn in range(10):+ messages = messages + [+ {"role": "user", "content": f"turn {turn}"},+ {"role": "assistant", "content": "ok"},+ ]+ body = {+ "model": "claude-sonnet-4-5",+ "system": [+ {+ "type": "text",+ "text": "You are a coding agent.",+ "cache_control": {"type": "ephemeral"},+ }+ ],+ "messages": messages,+ }+ shape_request(body, settings, level_override=level)+ systems.append(_json.dumps(body["system"], sort_keys=True))++ assert len(set(systems)) == 1, "steering block must not move between turns"+ # And the client's own breakpoint is still the FIRST block, so the+ # prefix it marks is untouched by the appended steering.+ first = _json.loads(systems[0])+ assert first[0]["cache_control"] == {"type": "ephemeral"}+ assert first[-1]["text"].startswith("<headroom_output_shaping>")def test_effort_routing_survives_cache_mode(self):"""The savings that do not touch the cache key must still apply."""diff --git a/tests/test_output_shaper_responses.py b/tests/test_output_shaper_responses.pyindex 7ff881d3d..ae0a16e37 100644--- a/tests/test_output_shaper_responses.py+++ b/tests/test_output_shaper_responses.py@@ -15,6 +15,7 @@ from typing import Anyfrom headroom.proxy.handlers.openai import _shape_openai_responses_payloadfrom headroom.proxy.output_savings import conversation_key_from_responses_bodyfrom headroom.proxy.output_shaper import (+ DEFAULT_VERBOSITY_LEVEL,OutputShaperSettings,TurnKind,apply_responses_verbosity_steering,@@ -202,7 +203,7 @@ class TestShapeResponsesRequest:body = _mechanical_body()result = shape_responses_request(body, OutputShaperSettings(enabled=True))assert result.changed is True- assert "output_shaper:verbosity:L3" in (result.labels or [])+ assert f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}" in (result.labels or [])assert body["reasoning"]["effort"] == "high", "reasoning.effort is left alone now"assert body["instructions"].startswith("You are Codex.")@@ -217,7 +218,7 @@ class TestShapeResponsesRequest:result = shape_responses_request(body, OutputShaperSettings(enabled=True))assert result.changed is Trueassert body["reasoning"]["effort"] == "high"- assert result.labels == ["output_shaper:verbosity:L3"]+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]# ---------------------------------------------------------------------------diff --git a/tests/test_stats_output_reduction_gate.py b/tests/test_stats_output_reduction_gate.pyindex 8f9f70df1..35a9a2481 100644--- a/tests/test_stats_output_reduction_gate.py+++ b/tests/test_stats_output_reduction_gate.py@@ -218,10 +218,22 @@ def test_stats_gates_a_controller_verbosity_zero(tmp_path, monkeypatch):assert payload["method"] == "inactive"-def test_stats_gates_cache_mode_even_with_a_level_set(tmp_path, monkeypatch):- """``mode="cache"`` forces the resolved level to 0 in the handlers. /stats- omitted ``steering_allowed_for``, so it kept publishing the ledger."""+def test_stats_publishes_in_cache_mode_when_the_level_is_pinned(tmp_path, monkeypatch):+ """A pinned level runs in cache mode, so /stats must report it as live.++ /stats builds the SAME settings the handlers do; when they steer, it+ publishes. The inverse -- publishing a ledger no live steering produced --+ is what the sibling tests below guard."""monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")- assert payload["active"] is False- assert payload["method"] == "inactive"+ assert payload["active"] is True+ assert payload["method"] == "measured"+++def test_stats_publishes_in_cache_mode_with_the_shaper_alone(tmp_path, monkeypatch):+ """An enabled shaper steers at the default level in cache mode, so /stats+ must report it live -- with no HEADROOM_VERBOSITY_LEVEL set at all."""+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)+ payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")+ assert payload["active"] is True+ assert payload["method"] == "measured"