Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): make the output shaper actually reduce output in the default mode (#3773)

Tejas Choprac02929846caf
diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.py
index c0989a8d9..39e6f6c45 100644
--- a/headroom/proxy/output_shaper.py
+++ b/headroom/proxy/output_shaper.py
@@ -62,6 +62,7 @@ from headroom.rollout import FeatureDecisionReason
logger = logging.getLogger(__name__)
__all__ = [
+ "DEFAULT_VERBOSITY_LEVEL",
"OutputShaperSettings",
"ShapeResult",
"TurnKind",
@@ -80,12 +81,23 @@ __all__ = [
_replace_or_append_steering_block = replace_or_append_steering_block
+#: Level handed to an operator who turned the shaper on and said nothing else.
+#: Two, not three: L3 instructs the model to drop content ("give conclusions
+#: only; omit rationale"), which reads as the model getting worse to someone who
+#: did not ask for it -- the same reasoning the feature's own rollout spec gives
+#: for not defaulting the shaper on at all. L2 only forbids ceremony and
+#: restatement. Measured on a long explain-with-code turn (3 paired runs,
+#: non-overlapping ranges): 2375 -> 1663 output tokens, -30%, with no visible
+#: loss of content.
+DEFAULT_VERBOSITY_LEVEL = 2
+
+
@dataclass(frozen=True)
class OutputShaperSettings:
"""Output-shaping settings with rollout enablement injected by the proxy."""
enabled: bool = False
- verbosity_level: int = 3
+ verbosity_level: int = DEFAULT_VERBOSITY_LEVEL
# False in ``mode="cache"``. Steering is the one lever that writes into the
# provider prefix-cache key (it appends to the system-prompt tail, and on a
# body with no system field it creates one); effort routing and the
@@ -110,9 +122,11 @@ class OutputShaperSettings:
"yes",
)
try:
- level = int(runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL", "3"))
+ level = int(
+ runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL", str(DEFAULT_VERBOSITY_LEVEL))
+ )
except ValueError:
- level = 2
+ level = DEFAULT_VERBOSITY_LEVEL
level = max(0, min(4, level))
return cls(
enabled=enabled,
@@ -165,37 +179,41 @@ def steering_allowed_for(config: Any) -> bool:
``mode="cache"`` freezes prior turns specifically to keep the provider's
prefix-cache key byte-stable (see ``ProxyConfig.mode``). Verbosity steering
- writes into that key, so running it there trades a large, certain cache
- cost for a small, uncertain output saving — the wrong side of a roughly
- 60x margin on a long context. Effort routing and the thinking budget are
- unaffected: they ride request parameters outside the cache key, so they
- keep saving in cache mode.
+ writes into that key, so a block that CHANGES there trades a large, certain
+ cache cost for a small, uncertain output saving — the wrong side of a
+ roughly 60x margin on a long context. Effort routing and the thinking
+ budget are unaffected: they ride request parameters outside the cache key,
+ so they keep saving in cache mode.
+
+ This is the gate for the sources that can change mid-conversation. It is
+ NOT the last word: :func:`resolve_verbosity_level` lets an explicitly
+ pinned ``HEADROOM_VERBOSITY_LEVEL`` through even here, because a level
+ fixed before startup is written into turn 1's prefix and never moves. See
+ that function for the reasoning.
"""
return getattr(config, "mode", None) != "cache"
-def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:
- """Resolve the live verbosity level and its source.
+#: Keys already warned about. Resolution runs on every request of every
+#: conversation, so an unguarded warning would be one log line per turn.
+_REPORTED: set[str] = set()
- Precedence:
- 1. ``HEADROOM_VERBOSITY_LEVEL`` set explicitly → manual override.
- 2. AIMD controller state (when ``HEADROOM_VERBOSITY_AUTOTUNE`` is on).
- 3. Learned ``verbosity.json`` from ``learn --verbosity``.
- 4. The settings default.
- Returns ``(level, source)``. Kept separate from :func:`shape_request` so the
- body-mutating core stays a pure function of an explicit level.
- """
- if not settings.steering_enabled:
- # Level 0 is the documented "no steering" value, so this disables the
- # only cache-key-mutating lever while leaving effort routing on. It
- # deliberately outranks the manual override below: a level set in the
- # environment must not reintroduce a prefix mutation the mode exists
- # to prevent.
- return 0, "cache_mode"
- if runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL"):
- return settings.verbosity_level, "env"
+def _report_once(key: str, message: str, *args: Any) -> None:
+ """Warn the first time only. Cleared by tests via ``_REPORTED.clear()``."""
+ if key in _REPORTED:
+ return
+ _REPORTED.add(key)
+ logger.warning(message, *args)
+
+def _resolve_unpinned_level(settings: OutputShaperSettings) -> tuple[int, str]:
+ """Resolve the level from the sources that can move under our feet.
+
+ The controller rewrites its state file as it hunts, and ``verbosity.json``
+ appears the moment someone runs ``learn --verbosity``. Both can therefore
+ change mid-conversation, which is what makes them unsafe in cache mode.
+ """
try:
from ..paths import workspace_dir
@@ -231,6 +249,83 @@ def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:
return settings.verbosity_level, "default"
+def resolve_verbosity_level(settings: OutputShaperSettings) -> tuple[int, str]:
+ """Resolve the live verbosity level and its source.
+
+ Precedence:
+ 1. ``HEADROOM_VERBOSITY_LEVEL`` set explicitly → manual override.
+ 2. AIMD controller state (when ``HEADROOM_VERBOSITY_AUTOTUNE`` is on).
+ 3. Learned ``verbosity.json`` from ``learn --verbosity``.
+ 4. The settings default.
+
+ Returns ``(level, source)``. Kept separate from :func:`shape_request` so the
+ body-mutating core stays a pure function of an explicit level.
+
+ Cache mode and the pinned level
+ -------------------------------
+ ``mode="cache"`` used to force this to ``0`` unconditionally, outranking
+ even an explicit ``HEADROOM_VERBOSITY_LEVEL``. That guarded the right
+ hazard with the wrong instrument.
+
+ What costs a prefix cache is the steering block *changing* — appearing on a
+ conversation already in flight, or moving between levels — because the
+ block sits in the ``system`` array, which is inside the cached prefix of
+ every message-level ``cache_control`` breakpoint the client sets. What does
+ NOT cost anything is the block simply *being there*: :func:`shape_request`
+ applies it on every request with no turn-kind gating, and the text is
+ byte-stable per level and idempotent via the sentinel. So a level pinned in
+ the environment before the proxy starts is written into turn 1's prefix and
+ is byte-identical on every turn after it. The cache is established
+ including the block and hits normally from then on; the only cost is the
+ block's own tokens riding along in the cached prefix (~80 at L2, about
+ $0.000024 per turn at Sonnet cache-read rates, against a measured ~700
+ output tokens saved per turn — roughly 450x the other way).
+
+ What matters is therefore not WHICH level, but whether it can move. Both
+ sources that cannot move are allowed through in cache mode:
+
+ * an explicit ``HEADROOM_VERBOSITY_LEVEL`` → ``env_pinned``
+ * otherwise the settings level, fixed at startup → ``cache_mode_default``
+
+ The second is what makes ``HEADROOM_OUTPUT_SHAPER=1`` sufficient on its
+ own. The shaper is opt-in (see the ``proxy_output_shaper`` rollout spec, which
+ deliberately does not default-enable it), so an enabled shaper is already an
+ explicit request and there is nothing further to ask the operator for.
+
+ The controller and the learned profile stay out of cache mode entirely —
+ those are exactly the sources that rewrite themselves while conversations
+ are open. Skipping them also spares the default mode two filesystem stats
+ per request.
+ """
+ if runtime_env.getenv("HEADROOM_VERBOSITY_LEVEL"):
+ return settings.verbosity_level, ("env" if settings.steering_enabled else "env_pinned")
+
+ if not settings.steering_enabled:
+ # Cache mode. The shaper is opt-in (see the ``proxy_output_shaper``
+ # rollout spec), so an enabled shaper is already an explicit operator
+ # act -- there is nothing further to ask for. Steer at the settings
+ # level, which was fixed at startup and cannot move.
+ #
+ # The controller and the learned profile are deliberately NOT consulted
+ # here. Both rewrite their files while conversations are open, and a
+ # level that moves mid-conversation rewrites the system array, which
+ # sits inside the cached prefix of every message-level cache_control
+ # breakpoint. That is the bust this mode exists to prevent -- and
+ # skipping the lookup also spares the default mode two filesystem stats
+ # per request.
+ if runtime_env.getenv("HEADROOM_VERBOSITY_AUTOTUNE", "").lower() in ("1", "true", "yes"):
+ _report_once(
+ "autotune_in_cache_mode",
+ "OutputShaper: HEADROOM_VERBOSITY_AUTOTUNE is set but the AIMD controller is "
+ "not consulted in cache mode — a level that moves mid-conversation busts the "
+ "prefix cache. Steering at L%d. Set HEADROOM_MODE=token to autotune.",
+ settings.verbosity_level,
+ )
+ return settings.verbosity_level, "cache_mode_default"
+
+ return _resolve_unpinned_level(settings)
+
+
@dataclass
class ShapeResult:
"""What the shaper did to a request body."""
diff --git a/tests/gateway/test_output_steering.py b/tests/gateway/test_output_steering.py
index 852a66365..565f90081 100644
--- a/tests/gateway/test_output_steering.py
+++ b/tests/gateway/test_output_steering.py
@@ -209,16 +209,22 @@ def test_env_off_wins(make_headroom_client, monkeypatch) -> None:
assert data["body"]["system"] == SYSTEM_PROMPT
-def test_cache_mode_labels_but_never_steers(make_headroom_client, monkeypatch) -> None:
- """``steering_allowed_for`` is false in ``mode="cache"``: the measurement
- label still rides the outcome but the prefix-cache key is left alone."""
+def test_cache_mode_steers_at_the_default_level(make_headroom_client, monkeypatch) -> None:
+ """``HEADROOM_OUTPUT_SHAPER=1`` alone must steer, in the default mode.
+
+ Cache mode used to resolve the level to 0 here, so the label rode the
+ outcome while the body went out untouched -- the feature did nothing for
+ the mode almost every deployment runs. A level fixed at startup cannot
+ move mid-conversation, so it cannot bust the prefix cache, so it steers.
+ """
monkeypatch.setenv("HEADROOM_OUTPUT_SHAPER", "1")
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
client = make_headroom_client(mode="cache")
for sent in (_anthropic_body(gateway={}), _openai_body(gateway={})):
data = compress(client, sent).json()
assert _stratum_label(data).startswith(STRATUM_PREFIX)
- assert not any(t.startswith(VERBOSITY_PREFIX) for t in data["transforms_applied"])
- assert STEERING_SENTINEL not in canonical(data["body"])
+ assert any(t.startswith(VERBOSITY_PREFIX) for t in data["transforms_applied"])
+ assert STEERING_SENTINEL in canonical(data["body"])
def test_control_arm_labels_but_never_steers(make_headroom_client, monkeypatch) -> None:
diff --git a/tests/test_output_shaper.py b/tests/test_output_shaper.py
index 0c08d68f9..a609db09c 100644
--- a/tests/test_output_shaper.py
+++ b/tests/test_output_shaper.py
@@ -10,6 +10,7 @@ import copy
from typing import Any
from headroom.proxy.output_shaper import (
+ DEFAULT_VERBOSITY_LEVEL,
OutputShaperSettings,
TurnKind,
apply_openai_responses_verbosity_steering,
@@ -180,10 +181,10 @@ class TestShapeRequest:
}
result = shape_request(body, ENABLED)
assert result.changed is True
- assert result.labels == ["output_shaper:verbosity:L3"]
+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]
assert body["output_config"]["effort"] == "xhigh", "must not touch effort"
assert body["thinking"] == {"type": "adaptive"}, "must not touch thinking"
- assert body["system"][1]["text"] == steering_text(3)
+ assert body["system"][1]["text"] == steering_text(DEFAULT_VERBOSITY_LEVEL)
def test_new_ask_gets_steering_but_keeps_effort(self):
body = {
@@ -192,7 +193,7 @@ class TestShapeRequest:
"output_config": {"effort": "xhigh"},
}
result = shape_request(body, ENABLED)
- assert result.labels == ["output_shaper:verbosity:L3"]
+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]
assert body["output_config"]["effort"] == "xhigh"
def test_second_pass_is_stable(self):
@@ -282,8 +283,8 @@ class TestShapeOpenAIChatRequest:
}
result = shape_openai_chat_request(body, ENABLED)
assert result.changed is True
- assert result.labels == ["output_shaper:verbosity:L3"]
- assert steering_text(3) in body["messages"][0]["content"]
+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]
+ assert steering_text(DEFAULT_VERBOSITY_LEVEL) in body["messages"][0]["content"]
# User turn is untouched.
assert body["messages"][1] == {"role": "user", "content": "hi"}
@@ -398,20 +399,144 @@ class TestCacheModeSuppressesSteeringOnly:
assert steering_allowed_for(SimpleNamespace(mode="cache")) is False
assert steering_allowed_for(None) is True, "absent config must not disable levers"
- def test_cache_mode_resolves_level_zero(self):
+ def test_cache_mode_steers_at_the_startup_level(self):
+ """Cache mode used to force 0 here. What it must prevent is a level that
+ MOVES; a level fixed at startup cannot, so it is steered at."""
from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level
settings = OutputShaperSettings(enabled=True, verbosity_level=3, steering_enabled=False)
- assert resolve_verbosity_level(settings) == (0, "cache_mode")
+ assert resolve_verbosity_level(settings) == (3, "cache_mode_default")
- def test_cache_mode_outranks_the_manual_level_override(self, monkeypatch):
- """An env-set level must not reintroduce the prefix mutation."""
+ def test_cache_mode_honours_a_pinned_manual_level(self, monkeypatch):
+ """A level pinned before startup never moves, so it cannot bust a cache.
+
+ The block lands in turn 1's prefix and is byte-identical on every turn
+ after it, so the cached prefix is established WITH it and hits normally.
+ Previously this resolved to 0 and the knob was silently ignored.
+ """
from headroom.proxy import runtime_env
from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level
monkeypatch.setattr(runtime_env, "getenv", lambda k, d="": "4" if "VERBOSITY" in k else d)
settings = OutputShaperSettings(enabled=True, verbosity_level=4, steering_enabled=False)
- assert resolve_verbosity_level(settings)[0] == 0
+ assert resolve_verbosity_level(settings) == (4, "env_pinned")
+
+ def test_shaper_alone_steers_at_l2_in_cache_mode(self, monkeypatch):
+ """``HEADROOM_OUTPUT_SHAPER=1`` must be sufficient on its own.
+
+ The shaper is opt-in, so an enabled shaper is already an explicit
+ request; there is nothing further to ask the operator for. Previously
+ this resolved to 0 and the feature did nothing in the default mode.
+ """
+ from headroom.proxy import runtime_env
+ from headroom.proxy.output_shaper import (
+ DEFAULT_VERBOSITY_LEVEL,
+ OutputShaperSettings,
+ resolve_verbosity_level,
+ )
+
+ monkeypatch.setattr(runtime_env, "getenv", lambda k, d="": d)
+ settings = OutputShaperSettings.from_env(enabled=True, steering_enabled=False)
+ assert settings.enabled is True
+ assert resolve_verbosity_level(settings) == (DEFAULT_VERBOSITY_LEVEL, "cache_mode_default")
+ assert DEFAULT_VERBOSITY_LEVEL == 2
+
+ def test_cache_mode_ignores_a_learned_level(self, tmp_path, monkeypatch):
+ """``verbosity.json`` appears the moment someone runs ``learn``, so it
+ must not be consulted where a mid-conversation change busts a cache."""
+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
+ (tmp_path / "verbosity.json").write_text('{"verbosity_level": 4}')
+
+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)
+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")
+
+ def test_cache_mode_ignores_the_controller_and_says_so_once(
+ self, tmp_path, monkeypatch, caplog
+ ):
+ """Autotune silently doing nothing is invisible from outside."""
+ import logging
+
+ from headroom.proxy import output_shaper
+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ monkeypatch.setenv("HEADROOM_VERBOSITY_AUTOTUNE", "1")
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
+ (tmp_path / "verbosity_controller.json").write_text('{"level": 4}')
+ output_shaper._REPORTED.clear()
+
+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)
+ with caplog.at_level(logging.WARNING, logger="headroom.proxy.output_shaper"):
+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")
+ resolve_verbosity_level(settings)
+
+ warnings = [r for r in caplog.records if "AUTOTUNE" in r.getMessage()]
+ assert len(warnings) == 1, "must not reprint on every request"
+ assert "HEADROOM_MODE=token" in warnings[0].getMessage()
+
+ def test_cache_mode_reads_no_workspace_files(self, monkeypatch):
+ """Resolution runs per request; the default mode must not stat files."""
+ import headroom.paths as paths
+ from headroom.proxy.output_shaper import OutputShaperSettings, resolve_verbosity_level
+
+ def _boom():
+ raise AssertionError("workspace_dir() must not be consulted in cache mode")
+
+ monkeypatch.setattr(paths, "workspace_dir", _boom)
+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)
+ assert resolve_verbosity_level(settings) == (2, "cache_mode_default")
+
+ def test_pinned_level_keeps_the_system_array_byte_stable_across_turns(self, monkeypatch):
+ """The cache-safety claim, asserted rather than argued.
+
+ Ten turns of a growing conversation must produce a byte-identical
+ ``system`` array -- that identity is the whole reason a pinned level
+ costs no cache.
+ """
+ import json as _json
+
+ from headroom.proxy import runtime_env
+ from headroom.proxy.output_shaper import (
+ OutputShaperSettings,
+ resolve_verbosity_level,
+ shape_request,
+ )
+
+ monkeypatch.setattr(runtime_env, "getenv", lambda k, d="": "2" if "VERBOSITY" in k else d)
+ settings = OutputShaperSettings(enabled=True, verbosity_level=2, steering_enabled=False)
+ level, source = resolve_verbosity_level(settings)
+ assert (level, source) == (2, "env_pinned")
+
+ systems = []
+ messages = []
+ for turn in range(10):
+ messages = messages + [
+ {"role": "user", "content": f"turn {turn}"},
+ {"role": "assistant", "content": "ok"},
+ ]
+ body = {
+ "model": "claude-sonnet-4-5",
+ "system": [
+ {
+ "type": "text",
+ "text": "You are a coding agent.",
+ "cache_control": {"type": "ephemeral"},
+ }
+ ],
+ "messages": messages,
+ }
+ shape_request(body, settings, level_override=level)
+ systems.append(_json.dumps(body["system"], sort_keys=True))
+
+ assert len(set(systems)) == 1, "steering block must not move between turns"
+ # And the client's own breakpoint is still the FIRST block, so the
+ # prefix it marks is untouched by the appended steering.
+ first = _json.loads(systems[0])
+ assert first[0]["cache_control"] == {"type": "ephemeral"}
+ assert first[-1]["text"].startswith("<headroom_output_shaping>")
def test_effort_routing_survives_cache_mode(self):
"""The savings that do not touch the cache key must still apply."""
diff --git a/tests/test_output_shaper_responses.py b/tests/test_output_shaper_responses.py
index 7ff881d3d..ae0a16e37 100644
--- a/tests/test_output_shaper_responses.py
+++ b/tests/test_output_shaper_responses.py
@@ -15,6 +15,7 @@ from typing import Any
from headroom.proxy.handlers.openai import _shape_openai_responses_payload
from headroom.proxy.output_savings import conversation_key_from_responses_body
from headroom.proxy.output_shaper import (
+ DEFAULT_VERBOSITY_LEVEL,
OutputShaperSettings,
TurnKind,
apply_responses_verbosity_steering,
@@ -202,7 +203,7 @@ class TestShapeResponsesRequest:
body = _mechanical_body()
result = shape_responses_request(body, OutputShaperSettings(enabled=True))
assert result.changed is True
- assert "output_shaper:verbosity:L3" in (result.labels or [])
+ assert f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}" in (result.labels or [])
assert body["reasoning"]["effort"] == "high", "reasoning.effort is left alone now"
assert body["instructions"].startswith("You are Codex.")
@@ -217,7 +218,7 @@ class TestShapeResponsesRequest:
result = shape_responses_request(body, OutputShaperSettings(enabled=True))
assert result.changed is True
assert body["reasoning"]["effort"] == "high"
- assert result.labels == ["output_shaper:verbosity:L3"]
+ assert result.labels == [f"output_shaper:verbosity:L{DEFAULT_VERBOSITY_LEVEL}"]
# ---------------------------------------------------------------------------
diff --git a/tests/test_stats_output_reduction_gate.py b/tests/test_stats_output_reduction_gate.py
index 8f9f70df1..35a9a2481 100644
--- a/tests/test_stats_output_reduction_gate.py
+++ b/tests/test_stats_output_reduction_gate.py
@@ -218,10 +218,22 @@ def test_stats_gates_a_controller_verbosity_zero(tmp_path, monkeypatch):
assert payload["method"] == "inactive"
-def test_stats_gates_cache_mode_even_with_a_level_set(tmp_path, monkeypatch):
- """``mode="cache"`` forces the resolved level to 0 in the handlers. /stats
- omitted ``steering_allowed_for``, so it kept publishing the ledger."""
+def test_stats_publishes_in_cache_mode_when_the_level_is_pinned(tmp_path, monkeypatch):
+ """A pinned level runs in cache mode, so /stats must report it as live.
+
+ /stats builds the SAME settings the handlers do; when they steer, it
+ publishes. The inverse -- publishing a ledger no live steering produced --
+ is what the sibling tests below guard."""
monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")
payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")
- assert payload["active"] is False
- assert payload["method"] == "inactive"
+ assert payload["active"] is True
+ assert payload["method"] == "measured"
+
+
+def test_stats_publishes_in_cache_mode_with_the_shaper_alone(tmp_path, monkeypatch):
+ """An enabled shaper steers at the default level in cache mode, so /stats
+ must report it live -- with no HEADROOM_VERBOSITY_LEVEL set at all."""
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
+ payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")
+ assert payload["active"] is True
+ assert payload["method"] == "measured"