Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(stats): gate the output-shaping claim on the shaper being active (#3386)

gglucass074f0ae9df50
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py
index edb4fcc26..0b87c0039 100644
--- a/headroom/proxy/output_savings.py
+++ b/headroom/proxy/output_savings.py
@@ -333,6 +333,11 @@ class SavingsEstimate:
return asdict(self)
+# Emitted by ``output_shaper.shape_request`` only when it actually changed the
+# request, so its presence is the per-request proof that shaping happened.
+_SHAPED_LABEL_PREFIX = "output_shaper:verbosity:"
+
+
@dataclass
class SavingsLedger:
"""Accumulates shaped (treatment) and unshaped (control) observations and
@@ -529,6 +534,10 @@ class SavingsLedger:
def to_dict(self) -> dict[str, Any]:
return {
+ # Marks arms accumulated under the shaped-only recording rule (see
+ # ``record_from_labels``). Absent = written before that rule, so the
+ # arms may hold unshaped observations.
+ "shaped_only": True,
"baseline": self.baseline.to_dict(),
"treatment": {k: a.to_dict() for k, a in self.treatment.items()},
"control": {k: a.to_dict() for k, a in self.control.items()},
@@ -537,6 +546,21 @@ class SavingsLedger:
@classmethod
def from_dict(cls, d: dict[str, Any]) -> SavingsLedger:
ledger = cls(baseline=BaselineModel.from_dict(d.get("baseline") or {}))
+ if not d.get("shaped_only"):
+ # Pre-rule arms cannot be told apart from shaped ones entry by
+ # entry, and republishing them is the reported bug. Drop them and
+ # re-accumulate from live traffic (hours, not weeks). The offline
+ # baseline is kept: it is learned from pre-shaper history, costs a
+ # `learn --verbosity` run to rebuild, and was never the poisoned part.
+ if d.get("treatment") or d.get("control"):
+ logger.warning(
+ "output-savings ledger predates shaped-only recording; "
+ "dropping %d treatment and %d control strata and "
+ "re-accumulating (baseline kept)",
+ len(d.get("treatment") or {}),
+ len(d.get("control") or {}),
+ )
+ return ledger
for k, a in (d.get("treatment") or {}).items():
ledger.treatment[k] = _Accum.from_dict(a)
for k, a in (d.get("control") or {}).items():
@@ -607,9 +631,10 @@ class SavingsRecorder:
adopted it) still records its output tokens; it just does not advance
the stratum's cluster count.
"""
+ label_strings = tuple(str(label) for label in labels or ())
arm_key: tuple[str, str] | None = None
conversation: str | None = None
- for label in labels or ():
+ for label in label_strings:
text = str(label)
if arm_key is None:
arm_key = parse_stratum_label(text)
@@ -620,6 +645,10 @@ class SavingsRecorder:
if arm_key is None:
return False
arm, key = arm_key
+ if arm == "treatment" and not any(
+ str(label).startswith(_SHAPED_LABEL_PREFIX) for label in label_strings
+ ):
+ return False
with self._lock:
self._ledger.record(arm, key, output_tokens, conversation)
self._since_flush += 1
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 4895a25ff..e06a8a1ce 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -476,6 +476,58 @@ logging.basicConfig(
logger = logging.getLogger("headroom.proxy")
+def _output_reduction_payload(shaper_active: bool, est: Any = None) -> dict[str, Any]:
+ """/stats payload for the output-shaping layer.
+
+ A deployment whose shaper is off (rollout-blocked channel, kill switch,
+ disabled config) is shaping nothing, so /stats must not publish the
+ persisted ledger's estimate as this deployment's live layer: the ledger
+ survives restarts and can span periods or key schemas where arms
+ accumulated unshaped traffic — which is how a rollout-blocked stable
+ install came to advertise a "measured" -147.9% output reduction it never
+ performed. The ledger itself stays untouched; only the claim is gated.
+
+ Every branch emits the SAME key set, nulls where a value does not exist,
+ so a consumer reading ``ci_low_percent`` or ``band_is_ci`` cannot KeyError
+ on the inactive or no-data paths. Numeric totals stay 0 rather than null:
+ callers already sum them.
+ """
+ payload: dict[str, Any] = {
+ "available": False,
+ "active": bool(shaper_active),
+ "method": None,
+ "band_is_ci": None,
+ "tokens_saved": 0,
+ "baseline_tokens": 0,
+ "reduction_percent": 0.0,
+ "ci_low_percent": None,
+ "ci_high_percent": None,
+ "requests": 0,
+ }
+ if not shaper_active:
+ payload["available"] = True
+ payload["method"] = "inactive"
+ return payload
+ if getattr(est, "n_requests", 0) <= 0:
+ return payload
+ payload.update(
+ {
+ "available": True,
+ "method": est.kind, # "measured" | "estimated" | "modelled"
+ # A modelled band is the spread between the two benchmarked
+ # models, not a sampling CI. The UI must not call it one.
+ "band_is_ci": est.kind != "modelled",
+ "tokens_saved": round(est.tokens_saved),
+ "baseline_tokens": round(est.baseline_tokens),
+ "reduction_percent": round(est.pct, 1),
+ "ci_low_percent": round(est.ci_low_pct, 1),
+ "ci_high_percent": round(est.ci_high_pct, 1),
+ "requests": est.n_requests,
+ }
+ )
+ return payload
+
+
def _code_syntax_breaker_status() -> dict[str, Any]:
"""Per-language AST-compression breaker state, or {} if unavailable."""
try:
@@ -4323,44 +4375,51 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# tokens the model didn't emit because we steered verbosity / routed
# effort down. Always labelled estimated-vs-measured + a CI so it's
# never mistaken for an exact count. Best-effort — never break /stats.
- output_reduction: dict[str, Any] = {"available": False}
+ # Same key set as every other branch: a consumer must not have to
+ # discriminate on which failure produced the payload.
+ output_reduction: dict[str, Any] = _output_reduction_payload(False)
+ output_reduction["available"] = False
+ output_reduction["method"] = None
try:
from headroom.proxy.output_savings import get_recorder
from headroom.proxy.output_shaper import (
OutputShaperSettings,
resolve_verbosity_level,
shaper_enabled_for,
+ steering_allowed_for,
)
# The active steering level enables the modelled fallback, which is
# what a deployment with no holdout and no learned baseline has --
# i.e. every fresh install, since `learn --verbosity` needs history
# that predates the shaper.
+ _shaper_active = False
_olevel: int | None = None
try:
+ _oconfig = getattr(proxy, "config", None)
_osettings = OutputShaperSettings.from_env(
- enabled=shaper_enabled_for(getattr(proxy, "config", None))
+ enabled=shaper_enabled_for(_oconfig),
+ # Cache mode forces the resolved level to 0. The request
+ # handlers pass this too; /stats has to build the SAME
+ # settings or it reads a level the handlers never used.
+ steering_enabled=steering_allowed_for(_oconfig),
)
if _osettings.enabled:
_olevel = resolve_verbosity_level(_osettings)[0]
+ # Steering is the only lever this layer's ledger measures,
+ # and level 0 is the documented "no steering" value -- the
+ # same test `shape_request` applies before it touches a
+ # body. An enabled shaper resolved to 0 (cache mode, an
+ # explicit HEADROOM_VERBOSITY_LEVEL=0, a learned or
+ # controller zero) shapes nothing, so it must not publish
+ # the persisted ledger as a live claim either.
+ _shaper_active = _olevel > 0
except Exception: # pragma: no cover - defensive
+ _shaper_active = False
_olevel = None
_oest = get_recorder().estimate(_olevel)
- if _oest.n_requests > 0:
- output_reduction = {
- "available": True,
- "method": _oest.kind, # "measured" | "estimated" | "modelled"
- # A modelled band is the spread between the two benchmarked
- # models, not a sampling CI. The UI must not call it one.
- "band_is_ci": _oest.kind != "modelled",
- "tokens_saved": round(_oest.tokens_saved),
- "baseline_tokens": round(_oest.baseline_tokens),
- "reduction_percent": round(_oest.pct, 1),
- "ci_low_percent": round(_oest.ci_low_pct, 1),
- "ci_high_percent": round(_oest.ci_high_pct, 1),
- "requests": _oest.n_requests,
- }
+ output_reduction = _output_reduction_payload(_shaper_active, _oest)
except Exception: # pragma: no cover - defensive
pass
diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py
index a78bf1ef0..6728f9c74 100644
--- a/tests/test_output_savings.py
+++ b/tests/test_output_savings.py
@@ -26,6 +26,11 @@ from headroom.proxy.output_savings import (
# ---------------------------------------------------------------------------
+# A treatment observation only counts when the request was actually shaped,
+# evidenced by the shaper's own verbosity label on the same channel.
+SHAPED = "output_shaper:verbosity:L2"
+
+
class TestStratification:
def test_input_buckets_monotone(self):
assert input_bucket(0) == "xs"
@@ -406,6 +411,7 @@ class TestHoldoutClusterGate:
assert recorder.record_from_labels(
[
"router:noop",
+ "output_shaper:verbosity:concise",
stratum_label("treatment", "opus|a|s|tools"),
conversation_label(key),
],
@@ -419,7 +425,9 @@ class TestHoldoutClusterGate:
def test_a_request_without_a_conversation_label_still_records(self, tmp_path):
recorder = SavingsRecorder(tmp_path / "savings.json", flush_every=1)
- assert recorder.record_from_labels([stratum_label("treatment", "opus|a|s|tools")], 800)
+ assert recorder.record_from_labels(
+ [stratum_label("treatment", "opus|a|s|tools"), "output_shaper:verbosity:concise"], 800
+ )
ledger = SavingsLedger.load(tmp_path / "savings.json")
assert ledger.treatment["opus|a|s|tools"].n == 1
assert ledger.treatment["opus|a|s|tools"].n_clusters == 0
@@ -434,6 +442,8 @@ class TestHoldoutClusterGate:
cluster gate exists to exclude -- and nothing on disk can say.
"""
return {
+ # Shaped-only arms can predate conversation provenance.
+ "shaped_only": True,
"baseline": {"strata": {}},
"treatment": {
"opus|a|s|tools": {
@@ -589,7 +599,7 @@ class TestRecorderBaselineReload:
recorder = SavingsRecorder(path, flush_every=1)
for output_tokens in (200, 210, 190):
- recorder.record_from_labels([stratum_label("treatment", key)], output_tokens)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], output_tokens)
# No baseline to compare against yet, so there is nothing to estimate.
assert recorder.estimate().n_requests == 0
@@ -620,7 +630,7 @@ class TestRecorderBaselineReload:
learned.save(path)
assert SavingsLedger.load(path).baseline.total_samples == 4
- recorder.record_from_labels([stratum_label("treatment", key)], 200)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 200)
recorder.flush()
# The flush must keep the learned baseline rather than writing the empty
@@ -649,7 +659,7 @@ class TestRecorderBaselineReload:
recorder = SavingsRecorder(path, flush_every=1)
for output_tokens in (200, 210, 190):
- recorder.record_from_labels([stratum_label("treatment", key)], output_tokens)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], output_tokens)
# First learn writes a baseline; the recorder adopts it.
first = SavingsLedger.load(path)
@@ -692,7 +702,7 @@ class TestFlushDurability:
key = SAMPLE_KEY
recorder = SavingsRecorder(path, flush_every=1)
- recorder.record_from_labels([stratum_label("treatment", key)], 200)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 200)
recorder.flush()
assert SavingsLedger.load(path).treatment[key].n == 1
@@ -700,7 +710,7 @@ class TestFlushDurability:
raise OSError(5, "simulated crash before rename")
monkeypatch.setattr(headroom.fsutil.os, "replace", _die_before_rename)
- recorder.record_from_labels([stratum_label("treatment", key)], 210)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 210)
recorder.flush() # OSError swallowed by the recorder — fail-open by design
# The pre-crash sample must survive and no temp residue may be left
@@ -758,7 +768,7 @@ class TestFlushDurability:
output_tokens=50,
tokens_saved=20,
attempted_input_tokens=100,
- transforms_applied=(stratum_label("treatment", SAMPLE_KEY),),
+ transforms_applied=(stratum_label("treatment", SAMPLE_KEY), SHAPED),
)
asyncio.run(emit_request_outcome(_Handler(), outcome))
diff --git a/tests/test_output_savings_cli.py b/tests/test_output_savings_cli.py
index 6440dec20..8c14ad60e 100644
--- a/tests/test_output_savings_cli.py
+++ b/tests/test_output_savings_cli.py
@@ -9,6 +9,10 @@ from click.testing import CliRunner
from headroom.cli.main import main
from headroom.proxy.output_savings import SavingsRecorder, stratum_label
+# A treatment observation only counts when the request was actually shaped,
+# evidenced by the shaper's own verbosity label on the same channel.
+SHAPED = "output_shaper:verbosity:L2"
+
def test_output_savings_empty(tmp_path, monkeypatch):
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
@@ -41,7 +45,11 @@ def test_recorder_round_trips_via_labels(tmp_path):
rec = SavingsRecorder(path, flush_every=1)
# Baseline so the estimate has something to compare against.
rec._ledger.baseline.observe("opus|new_user_ask|s|tools", 1000)
- labels = ["compress:smartcrush", stratum_label("treatment", "opus|new_user_ask|s|tools")]
+ labels = [
+ "compress:smartcrush",
+ stratum_label("treatment", "opus|new_user_ask|s|tools"),
+ SHAPED,
+ ]
assert rec.record_from_labels(labels, output_tokens=600) is True
assert rec.record_from_labels(["no-shaper-label"], output_tokens=999) is False
est = rec.estimate()
diff --git a/tests/test_stats_output_reduction_gate.py b/tests/test_stats_output_reduction_gate.py
new file mode 100644
index 000000000..8f9f70df1
--- /dev/null
+++ b/tests/test_stats_output_reduction_gate.py
@@ -0,0 +1,227 @@
+"""/stats must not publish output-shaping claims a deployment never performed.
+
+The output-savings ledger persists across restarts, and its arms can hold
+unshaped traffic from periods when the shaper was rollout-blocked or from
+pre-fix conversation-key schemas. A stable install with the shaper
+blocked_by_channel published a "measured" -147.9% output reduction straight
+from that ledger — a claim about shaping it never did. The /stats seam now
+gates the claim on the shaper being active; the ledger itself is untouched.
+"""
+
+from __future__ import annotations
+
+from headroom.proxy.output_savings import SavingsEstimate, SavingsLedger, SavingsRecorder
+from headroom.proxy.server import _output_reduction_payload
+
+
+def _poisoned_measured() -> SavingsEstimate:
+ # The audited real-world shape: blocked stable channel, lopsided legacy
+ # arms, "measured" -147.9% over 19,644 requests.
+ return SavingsEstimate(
+ tokens_saved=-7_522_490,
+ baseline_tokens=5_086_874,
+ pct=-147.9,
+ ci_low_pct=-160.2,
+ ci_high_pct=-135.6,
+ n_requests=19_644,
+ kind="measured",
+ )
+
+
+def test_inactive_shaper_claims_zero_regardless_of_ledger():
+ payload = _output_reduction_payload(False, _poisoned_measured())
+ assert payload["active"] is False
+ assert payload["method"] == "inactive"
+ assert payload["tokens_saved"] == 0
+ assert payload["reduction_percent"] == 0.0
+ assert payload["requests"] == 0
+ # Still an available, well-formed layer entry, so consumers can render
+ # "inactive" instead of mistaking absence for zero measurement.
+ assert payload["available"] is True
+
+
+def test_active_shaper_publishes_the_estimate_as_before():
+ payload = _output_reduction_payload(True, _poisoned_measured())
+ assert payload["active"] is True
+ assert payload["available"] is True
+ assert payload["method"] == "measured"
+ assert payload["band_is_ci"] is True
+ assert payload["tokens_saved"] == -7_522_490
+ assert payload["reduction_percent"] == -147.9
+ assert payload["requests"] == 19_644
+
+
+def test_active_shaper_without_data_stays_unavailable():
+ empty = SavingsEstimate(
+ tokens_saved=0.0,
+ baseline_tokens=0.0,
+ pct=0.0,
+ ci_low_pct=0.0,
+ ci_high_pct=0.0,
+ n_requests=0,
+ kind="estimated",
+ )
+ payload = _output_reduction_payload(True, empty)
+ assert payload["available"] is False
+ assert payload["active"] is True
+ assert payload["method"] is None
+
+
+def test_every_branch_emits_the_same_key_set():
+ """A consumer reading ci_low_percent or band_is_ci must not KeyError on the
+ inactive or no-data paths — the desktop app and dashboard both read these."""
+ no_data = SavingsEstimate(
+ tokens_saved=0.0,
+ baseline_tokens=0.0,
+ pct=0.0,
+ ci_low_pct=0.0,
+ ci_high_pct=0.0,
+ n_requests=0,
+ kind="estimated",
+ )
+ branches = [
+ _output_reduction_payload(False, _poisoned_measured()),
+ _output_reduction_payload(True, no_data),
+ _output_reduction_payload(True, _poisoned_measured()),
+ _output_reduction_payload(False),
+ ]
+ expected = set(branches[-1])
+ for payload in branches:
+ assert set(payload) == expected
+ # Numeric totals stay numeric on every branch; only the band is nullable.
+ assert isinstance(payload["tokens_saved"], int | float)
+ assert isinstance(payload["baseline_tokens"], int | float)
+ assert isinstance(payload["reduction_percent"], int | float)
+ assert isinstance(payload["requests"], int)
+
+
+def test_unshaped_treatment_request_is_not_recorded(tmp_path):
+ """The gate above fixes the install that reported it; this fixes the next
+ one. The stratum label is attached whenever the shaper is configured on,
+ but shaping can still be a no-op — and a treatment arm holding unshaped
+ output republishes the same bogus reduction the moment /stats unblocks."""
+ recorder = SavingsRecorder(path=tmp_path / "ledger.json", flush_every=1)
+
+ assert not recorder.record_from_labels(("output_shaper:stratum:opus|chat|m|tools",), 500)
+ assert recorder._ledger.treatment == {}
+
+ assert recorder.record_from_labels(
+ ("output_shaper:stratum:opus|chat|m|tools", "output_shaper:verbosity:L2"), 500
+ )
+ assert recorder._ledger.treatment["opus|chat|m|tools"].n == 1
+
+ # Control is unshaped by definition and keeps recording unconditionally.
+ assert recorder.record_from_labels(("output_shaper:control:opus|chat|m|tools",), 900)
+ assert recorder._ledger.control["opus|chat|m|tools"].n == 1
+
+
+def test_ledger_written_before_the_rule_drops_its_arms_but_keeps_the_baseline():
+ legacy = {
+ "baseline": {"strata": {"opus|chat|m|tools": {"mean": 800.0, "var": 100.0, "n": 40}}},
+ "treatment": {"opus|chat|m|tools": {"n": 19_644, "sum": 5_000_000.0, "sumsq": 1e12}},
+ "control": {"opus|chat|m|tools": {"n": 12, "sum": 4_000.0, "sumsq": 1e6}},
+ }
+ ledger = SavingsLedger.from_dict(legacy)
+ assert ledger.treatment == {}
+ assert ledger.control == {}
+ # The offline baseline costs a `learn --verbosity` run to rebuild and was
+ # never the poisoned part.
+ assert ledger.baseline.lookup("opus|chat|m|tools")[2] == 40
+ # Round-tripping now carries the marker, so it is kept next time.
+ assert SavingsLedger.from_dict(ledger.to_dict()).baseline.lookup("opus|chat|m|tools")[2] == 40
+
+
+# --- endpoint-level: the boolean /stats actually passes ----------------------
+#
+# The payload helper above is only half the gate. The other half is which
+# boolean the /stats route hands it, and that is derived from the shaper's
+# EFFECTIVE steering configuration — the same one the request handlers build.
+# An enabled shaper whose level resolves to 0 (cache mode, an explicit
+# override, a learned or controller zero) shapes nothing, so it must not
+# publish the persisted ledger either. Testing the helper alone cannot catch a
+# wrong boolean at the call site.
+
+
+def _live_estimate() -> SavingsEstimate:
+ return SavingsEstimate(
+ tokens_saved=200.0,
+ baseline_tokens=1000.0,
+ pct=20.0,
+ ci_low_pct=15.0,
+ ci_high_pct=25.0,
+ n_requests=5,
+ kind="measured",
+ )
+
+
+def _stats_output_reduction(tmp_path, monkeypatch, *, mode: str = "token"):
+ """Hit the real /stats route with a ledger that already holds an estimate."""
+ from fastapi.testclient import TestClient
+
+ import headroom.proxy.output_savings as output_savings
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))
+ monkeypatch.setenv("HEADROOM_OUTPUT_SHAPER", "1")
+
+ class _Recorder:
+ def estimate(self, level=None):
+ return _live_estimate()
+
+ monkeypatch.setattr(output_savings, "get_recorder", lambda: _Recorder())
+
+ config = ProxyConfig(
+ mode=mode,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ log_requests=False,
+ )
+ with TestClient(create_app(config)) as client:
+ response = client.get("/stats")
+ assert response.status_code == 200
+ return response.json()["tokens"]["output_reduction"]
+
+
+def test_stats_publishes_the_estimate_while_steering_is_live(tmp_path, monkeypatch):
+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")
+ payload = _stats_output_reduction(tmp_path, monkeypatch)
+ assert payload["active"] is True
+ assert payload["method"] == "measured"
+ assert payload["tokens_saved"] == 200
+ assert payload["reduction_percent"] == 20.0
+
+
+def test_stats_gates_an_explicit_verbosity_zero(tmp_path, monkeypatch):
+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "0")
+ payload = _stats_output_reduction(tmp_path, monkeypatch)
+ assert payload["active"] is False
+ assert payload["method"] == "inactive"
+ assert payload["tokens_saved"] == 0
+ assert payload["reduction_percent"] == 0.0
+
+
+def test_stats_gates_a_learned_verbosity_zero(tmp_path, monkeypatch):
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
+ (tmp_path / "verbosity.json").write_text('{"verbosity_level": 0}')
+ payload = _stats_output_reduction(tmp_path, monkeypatch)
+ assert payload["active"] is False
+ assert payload["method"] == "inactive"
+
+
+def test_stats_gates_a_controller_verbosity_zero(tmp_path, monkeypatch):
+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)
+ monkeypatch.setenv("HEADROOM_VERBOSITY_AUTOTUNE", "1")
+ (tmp_path / "verbosity_controller.json").write_text('{"level": 0}')
+ payload = _stats_output_reduction(tmp_path, monkeypatch)
+ assert payload["active"] is False
+ assert payload["method"] == "inactive"
+
+
+def test_stats_gates_cache_mode_even_with_a_level_set(tmp_path, monkeypatch):
+ """``mode="cache"`` forces the resolved level to 0 in the handlers. /stats
+ omitted ``steering_allowed_for``, so it kept publishing the ledger."""
+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")
+ payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")
+ assert payload["active"] is False
+ assert payload["method"] == "inactive"