Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(stats): gate the output-shaping claim on the shaper being active (#3386)
gglucass074f0ae9df50
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.pyindex edb4fcc26..0b87c0039 100644--- a/headroom/proxy/output_savings.py+++ b/headroom/proxy/output_savings.py@@ -333,6 +333,11 @@ class SavingsEstimate:return asdict(self)+# Emitted by ``output_shaper.shape_request`` only when it actually changed the+# request, so its presence is the per-request proof that shaping happened.+_SHAPED_LABEL_PREFIX = "output_shaper:verbosity:"++@dataclassclass SavingsLedger:"""Accumulates shaped (treatment) and unshaped (control) observations and@@ -529,6 +534,10 @@ class SavingsLedger:def to_dict(self) -> dict[str, Any]:return {+ # Marks arms accumulated under the shaped-only recording rule (see+ # ``record_from_labels``). Absent = written before that rule, so the+ # arms may hold unshaped observations.+ "shaped_only": True,"baseline": self.baseline.to_dict(),"treatment": {k: a.to_dict() for k, a in self.treatment.items()},"control": {k: a.to_dict() for k, a in self.control.items()},@@ -537,6 +546,21 @@ class SavingsLedger:@classmethoddef from_dict(cls, d: dict[str, Any]) -> SavingsLedger:ledger = cls(baseline=BaselineModel.from_dict(d.get("baseline") or {}))+ if not d.get("shaped_only"):+ # Pre-rule arms cannot be told apart from shaped ones entry by+ # entry, and republishing them is the reported bug. Drop them and+ # re-accumulate from live traffic (hours, not weeks). The offline+ # baseline is kept: it is learned from pre-shaper history, costs a+ # `learn --verbosity` run to rebuild, and was never the poisoned part.+ if d.get("treatment") or d.get("control"):+ logger.warning(+ "output-savings ledger predates shaped-only recording; "+ "dropping %d treatment and %d control strata and "+ "re-accumulating (baseline kept)",+ len(d.get("treatment") or {}),+ len(d.get("control") or {}),+ )+ return ledgerfor k, a in (d.get("treatment") or {}).items():ledger.treatment[k] = _Accum.from_dict(a)for k, a in (d.get("control") or {}).items():@@ -607,9 +631,10 @@ class SavingsRecorder:adopted it) still records its output tokens; it just does not advancethe stratum's cluster count."""+ label_strings = tuple(str(label) for label in labels or ())arm_key: tuple[str, str] | None = Noneconversation: str | None = None- for label in labels or ():+ for label in label_strings:text = str(label)if arm_key is None:arm_key = parse_stratum_label(text)@@ -620,6 +645,10 @@ class SavingsRecorder:if arm_key is None:return Falsearm, key = arm_key+ if arm == "treatment" and not any(+ str(label).startswith(_SHAPED_LABEL_PREFIX) for label in label_strings+ ):+ return Falsewith self._lock:self._ledger.record(arm, key, output_tokens, conversation)self._since_flush += 1diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 4895a25ff..e06a8a1ce 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -476,6 +476,58 @@ logging.basicConfig(logger = logging.getLogger("headroom.proxy")+def _output_reduction_payload(shaper_active: bool, est: Any = None) -> dict[str, Any]:+ """/stats payload for the output-shaping layer.++ A deployment whose shaper is off (rollout-blocked channel, kill switch,+ disabled config) is shaping nothing, so /stats must not publish the+ persisted ledger's estimate as this deployment's live layer: the ledger+ survives restarts and can span periods or key schemas where arms+ accumulated unshaped traffic — which is how a rollout-blocked stable+ install came to advertise a "measured" -147.9% output reduction it never+ performed. The ledger itself stays untouched; only the claim is gated.++ Every branch emits the SAME key set, nulls where a value does not exist,+ so a consumer reading ``ci_low_percent`` or ``band_is_ci`` cannot KeyError+ on the inactive or no-data paths. Numeric totals stay 0 rather than null:+ callers already sum them.+ """+ payload: dict[str, Any] = {+ "available": False,+ "active": bool(shaper_active),+ "method": None,+ "band_is_ci": None,+ "tokens_saved": 0,+ "baseline_tokens": 0,+ "reduction_percent": 0.0,+ "ci_low_percent": None,+ "ci_high_percent": None,+ "requests": 0,+ }+ if not shaper_active:+ payload["available"] = True+ payload["method"] = "inactive"+ return payload+ if getattr(est, "n_requests", 0) <= 0:+ return payload+ payload.update(+ {+ "available": True,+ "method": est.kind, # "measured" | "estimated" | "modelled"+ # A modelled band is the spread between the two benchmarked+ # models, not a sampling CI. The UI must not call it one.+ "band_is_ci": est.kind != "modelled",+ "tokens_saved": round(est.tokens_saved),+ "baseline_tokens": round(est.baseline_tokens),+ "reduction_percent": round(est.pct, 1),+ "ci_low_percent": round(est.ci_low_pct, 1),+ "ci_high_percent": round(est.ci_high_pct, 1),+ "requests": est.n_requests,+ }+ )+ return payload++def _code_syntax_breaker_status() -> dict[str, Any]:"""Per-language AST-compression breaker state, or {} if unavailable."""try:@@ -4323,44 +4375,51 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# tokens the model didn't emit because we steered verbosity / routed# effort down. Always labelled estimated-vs-measured + a CI so it's# never mistaken for an exact count. Best-effort — never break /stats.- output_reduction: dict[str, Any] = {"available": False}+ # Same key set as every other branch: a consumer must not have to+ # discriminate on which failure produced the payload.+ output_reduction: dict[str, Any] = _output_reduction_payload(False)+ output_reduction["available"] = False+ output_reduction["method"] = Nonetry:from headroom.proxy.output_savings import get_recorderfrom headroom.proxy.output_shaper import (OutputShaperSettings,resolve_verbosity_level,shaper_enabled_for,+ steering_allowed_for,)# The active steering level enables the modelled fallback, which is# what a deployment with no holdout and no learned baseline has --# i.e. every fresh install, since `learn --verbosity` needs history# that predates the shaper.+ _shaper_active = False_olevel: int | None = Nonetry:+ _oconfig = getattr(proxy, "config", None)_osettings = OutputShaperSettings.from_env(- enabled=shaper_enabled_for(getattr(proxy, "config", None))+ enabled=shaper_enabled_for(_oconfig),+ # Cache mode forces the resolved level to 0. The request+ # handlers pass this too; /stats has to build the SAME+ # settings or it reads a level the handlers never used.+ steering_enabled=steering_allowed_for(_oconfig),)if _osettings.enabled:_olevel = resolve_verbosity_level(_osettings)[0]+ # Steering is the only lever this layer's ledger measures,+ # and level 0 is the documented "no steering" value -- the+ # same test `shape_request` applies before it touches a+ # body. An enabled shaper resolved to 0 (cache mode, an+ # explicit HEADROOM_VERBOSITY_LEVEL=0, a learned or+ # controller zero) shapes nothing, so it must not publish+ # the persisted ledger as a live claim either.+ _shaper_active = _olevel > 0except Exception: # pragma: no cover - defensive+ _shaper_active = False_olevel = None_oest = get_recorder().estimate(_olevel)- if _oest.n_requests > 0:- output_reduction = {- "available": True,- "method": _oest.kind, # "measured" | "estimated" | "modelled"- # A modelled band is the spread between the two benchmarked- # models, not a sampling CI. The UI must not call it one.- "band_is_ci": _oest.kind != "modelled",- "tokens_saved": round(_oest.tokens_saved),- "baseline_tokens": round(_oest.baseline_tokens),- "reduction_percent": round(_oest.pct, 1),- "ci_low_percent": round(_oest.ci_low_pct, 1),- "ci_high_percent": round(_oest.ci_high_pct, 1),- "requests": _oest.n_requests,- }+ output_reduction = _output_reduction_payload(_shaper_active, _oest)except Exception: # pragma: no cover - defensivepassdiff --git a/tests/test_output_savings.py b/tests/test_output_savings.pyindex a78bf1ef0..6728f9c74 100644--- a/tests/test_output_savings.py+++ b/tests/test_output_savings.py@@ -26,6 +26,11 @@ from headroom.proxy.output_savings import (# ---------------------------------------------------------------------------+# A treatment observation only counts when the request was actually shaped,+# evidenced by the shaper's own verbosity label on the same channel.+SHAPED = "output_shaper:verbosity:L2"++class TestStratification:def test_input_buckets_monotone(self):assert input_bucket(0) == "xs"@@ -406,6 +411,7 @@ class TestHoldoutClusterGate:assert recorder.record_from_labels(["router:noop",+ "output_shaper:verbosity:concise",stratum_label("treatment", "opus|a|s|tools"),conversation_label(key),],@@ -419,7 +425,9 @@ class TestHoldoutClusterGate:def test_a_request_without_a_conversation_label_still_records(self, tmp_path):recorder = SavingsRecorder(tmp_path / "savings.json", flush_every=1)- assert recorder.record_from_labels([stratum_label("treatment", "opus|a|s|tools")], 800)+ assert recorder.record_from_labels(+ [stratum_label("treatment", "opus|a|s|tools"), "output_shaper:verbosity:concise"], 800+ )ledger = SavingsLedger.load(tmp_path / "savings.json")assert ledger.treatment["opus|a|s|tools"].n == 1assert ledger.treatment["opus|a|s|tools"].n_clusters == 0@@ -434,6 +442,8 @@ class TestHoldoutClusterGate:cluster gate exists to exclude -- and nothing on disk can say."""return {+ # Shaped-only arms can predate conversation provenance.+ "shaped_only": True,"baseline": {"strata": {}},"treatment": {"opus|a|s|tools": {@@ -589,7 +599,7 @@ class TestRecorderBaselineReload:recorder = SavingsRecorder(path, flush_every=1)for output_tokens in (200, 210, 190):- recorder.record_from_labels([stratum_label("treatment", key)], output_tokens)+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], output_tokens)# No baseline to compare against yet, so there is nothing to estimate.assert recorder.estimate().n_requests == 0@@ -620,7 +630,7 @@ class TestRecorderBaselineReload:learned.save(path)assert SavingsLedger.load(path).baseline.total_samples == 4- recorder.record_from_labels([stratum_label("treatment", key)], 200)+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 200)recorder.flush()# The flush must keep the learned baseline rather than writing the empty@@ -649,7 +659,7 @@ class TestRecorderBaselineReload:recorder = SavingsRecorder(path, flush_every=1)for output_tokens in (200, 210, 190):- recorder.record_from_labels([stratum_label("treatment", key)], output_tokens)+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], output_tokens)# First learn writes a baseline; the recorder adopts it.first = SavingsLedger.load(path)@@ -692,7 +702,7 @@ class TestFlushDurability:key = SAMPLE_KEYrecorder = SavingsRecorder(path, flush_every=1)- recorder.record_from_labels([stratum_label("treatment", key)], 200)+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 200)recorder.flush()assert SavingsLedger.load(path).treatment[key].n == 1@@ -700,7 +710,7 @@ class TestFlushDurability:raise OSError(5, "simulated crash before rename")monkeypatch.setattr(headroom.fsutil.os, "replace", _die_before_rename)- recorder.record_from_labels([stratum_label("treatment", key)], 210)+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 210)recorder.flush() # OSError swallowed by the recorder — fail-open by design# The pre-crash sample must survive and no temp residue may be left@@ -758,7 +768,7 @@ class TestFlushDurability:output_tokens=50,tokens_saved=20,attempted_input_tokens=100,- transforms_applied=(stratum_label("treatment", SAMPLE_KEY),),+ transforms_applied=(stratum_label("treatment", SAMPLE_KEY), SHAPED),)asyncio.run(emit_request_outcome(_Handler(), outcome))diff --git a/tests/test_output_savings_cli.py b/tests/test_output_savings_cli.pyindex 6440dec20..8c14ad60e 100644--- a/tests/test_output_savings_cli.py+++ b/tests/test_output_savings_cli.py@@ -9,6 +9,10 @@ from click.testing import CliRunnerfrom headroom.cli.main import mainfrom headroom.proxy.output_savings import SavingsRecorder, stratum_label+# A treatment observation only counts when the request was actually shaped,+# evidenced by the shaper's own verbosity label on the same channel.+SHAPED = "output_shaper:verbosity:L2"+def test_output_savings_empty(tmp_path, monkeypatch):monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))@@ -41,7 +45,11 @@ def test_recorder_round_trips_via_labels(tmp_path):rec = SavingsRecorder(path, flush_every=1)# Baseline so the estimate has something to compare against.rec._ledger.baseline.observe("opus|new_user_ask|s|tools", 1000)- labels = ["compress:smartcrush", stratum_label("treatment", "opus|new_user_ask|s|tools")]+ labels = [+ "compress:smartcrush",+ stratum_label("treatment", "opus|new_user_ask|s|tools"),+ SHAPED,+ ]assert rec.record_from_labels(labels, output_tokens=600) is Trueassert rec.record_from_labels(["no-shaper-label"], output_tokens=999) is Falseest = rec.estimate()diff --git a/tests/test_stats_output_reduction_gate.py b/tests/test_stats_output_reduction_gate.pynew file mode 100644index 000000000..8f9f70df1--- /dev/null+++ b/tests/test_stats_output_reduction_gate.py@@ -0,0 +1,227 @@+"""/stats must not publish output-shaping claims a deployment never performed.++The output-savings ledger persists across restarts, and its arms can hold+unshaped traffic from periods when the shaper was rollout-blocked or from+pre-fix conversation-key schemas. A stable install with the shaper+blocked_by_channel published a "measured" -147.9% output reduction straight+from that ledger — a claim about shaping it never did. The /stats seam now+gates the claim on the shaper being active; the ledger itself is untouched.+"""++from __future__ import annotations++from headroom.proxy.output_savings import SavingsEstimate, SavingsLedger, SavingsRecorder+from headroom.proxy.server import _output_reduction_payload+++def _poisoned_measured() -> SavingsEstimate:+ # The audited real-world shape: blocked stable channel, lopsided legacy+ # arms, "measured" -147.9% over 19,644 requests.+ return SavingsEstimate(+ tokens_saved=-7_522_490,+ baseline_tokens=5_086_874,+ pct=-147.9,+ ci_low_pct=-160.2,+ ci_high_pct=-135.6,+ n_requests=19_644,+ kind="measured",+ )+++def test_inactive_shaper_claims_zero_regardless_of_ledger():+ payload = _output_reduction_payload(False, _poisoned_measured())+ assert payload["active"] is False+ assert payload["method"] == "inactive"+ assert payload["tokens_saved"] == 0+ assert payload["reduction_percent"] == 0.0+ assert payload["requests"] == 0+ # Still an available, well-formed layer entry, so consumers can render+ # "inactive" instead of mistaking absence for zero measurement.+ assert payload["available"] is True+++def test_active_shaper_publishes_the_estimate_as_before():+ payload = _output_reduction_payload(True, _poisoned_measured())+ assert payload["active"] is True+ assert payload["available"] is True+ assert payload["method"] == "measured"+ assert payload["band_is_ci"] is True+ assert payload["tokens_saved"] == -7_522_490+ assert payload["reduction_percent"] == -147.9+ assert payload["requests"] == 19_644+++def test_active_shaper_without_data_stays_unavailable():+ empty = SavingsEstimate(+ tokens_saved=0.0,+ baseline_tokens=0.0,+ pct=0.0,+ ci_low_pct=0.0,+ ci_high_pct=0.0,+ n_requests=0,+ kind="estimated",+ )+ payload = _output_reduction_payload(True, empty)+ assert payload["available"] is False+ assert payload["active"] is True+ assert payload["method"] is None+++def test_every_branch_emits_the_same_key_set():+ """A consumer reading ci_low_percent or band_is_ci must not KeyError on the+ inactive or no-data paths — the desktop app and dashboard both read these."""+ no_data = SavingsEstimate(+ tokens_saved=0.0,+ baseline_tokens=0.0,+ pct=0.0,+ ci_low_pct=0.0,+ ci_high_pct=0.0,+ n_requests=0,+ kind="estimated",+ )+ branches = [+ _output_reduction_payload(False, _poisoned_measured()),+ _output_reduction_payload(True, no_data),+ _output_reduction_payload(True, _poisoned_measured()),+ _output_reduction_payload(False),+ ]+ expected = set(branches[-1])+ for payload in branches:+ assert set(payload) == expected+ # Numeric totals stay numeric on every branch; only the band is nullable.+ assert isinstance(payload["tokens_saved"], int | float)+ assert isinstance(payload["baseline_tokens"], int | float)+ assert isinstance(payload["reduction_percent"], int | float)+ assert isinstance(payload["requests"], int)+++def test_unshaped_treatment_request_is_not_recorded(tmp_path):+ """The gate above fixes the install that reported it; this fixes the next+ one. The stratum label is attached whenever the shaper is configured on,+ but shaping can still be a no-op — and a treatment arm holding unshaped+ output republishes the same bogus reduction the moment /stats unblocks."""+ recorder = SavingsRecorder(path=tmp_path / "ledger.json", flush_every=1)++ assert not recorder.record_from_labels(("output_shaper:stratum:opus|chat|m|tools",), 500)+ assert recorder._ledger.treatment == {}++ assert recorder.record_from_labels(+ ("output_shaper:stratum:opus|chat|m|tools", "output_shaper:verbosity:L2"), 500+ )+ assert recorder._ledger.treatment["opus|chat|m|tools"].n == 1++ # Control is unshaped by definition and keeps recording unconditionally.+ assert recorder.record_from_labels(("output_shaper:control:opus|chat|m|tools",), 900)+ assert recorder._ledger.control["opus|chat|m|tools"].n == 1+++def test_ledger_written_before_the_rule_drops_its_arms_but_keeps_the_baseline():+ legacy = {+ "baseline": {"strata": {"opus|chat|m|tools": {"mean": 800.0, "var": 100.0, "n": 40}}},+ "treatment": {"opus|chat|m|tools": {"n": 19_644, "sum": 5_000_000.0, "sumsq": 1e12}},+ "control": {"opus|chat|m|tools": {"n": 12, "sum": 4_000.0, "sumsq": 1e6}},+ }+ ledger = SavingsLedger.from_dict(legacy)+ assert ledger.treatment == {}+ assert ledger.control == {}+ # The offline baseline costs a `learn --verbosity` run to rebuild and was+ # never the poisoned part.+ assert ledger.baseline.lookup("opus|chat|m|tools")[2] == 40+ # Round-tripping now carries the marker, so it is kept next time.+ assert SavingsLedger.from_dict(ledger.to_dict()).baseline.lookup("opus|chat|m|tools")[2] == 40+++# --- endpoint-level: the boolean /stats actually passes ----------------------+#+# The payload helper above is only half the gate. The other half is which+# boolean the /stats route hands it, and that is derived from the shaper's+# EFFECTIVE steering configuration — the same one the request handlers build.+# An enabled shaper whose level resolves to 0 (cache mode, an explicit+# override, a learned or controller zero) shapes nothing, so it must not+# publish the persisted ledger either. Testing the helper alone cannot catch a+# wrong boolean at the call site.+++def _live_estimate() -> SavingsEstimate:+ return SavingsEstimate(+ tokens_saved=200.0,+ baseline_tokens=1000.0,+ pct=20.0,+ ci_low_pct=15.0,+ ci_high_pct=25.0,+ n_requests=5,+ kind="measured",+ )+++def _stats_output_reduction(tmp_path, monkeypatch, *, mode: str = "token"):+ """Hit the real /stats route with a ledger that already holds an estimate."""+ from fastapi.testclient import TestClient++ import headroom.proxy.output_savings as output_savings+ from headroom.proxy.server import ProxyConfig, create_app++ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))+ monkeypatch.setenv("HEADROOM_OUTPUT_SHAPER", "1")++ class _Recorder:+ def estimate(self, level=None):+ return _live_estimate()++ monkeypatch.setattr(output_savings, "get_recorder", lambda: _Recorder())++ config = ProxyConfig(+ mode=mode,+ cache_enabled=False,+ rate_limit_enabled=False,+ log_requests=False,+ )+ with TestClient(create_app(config)) as client:+ response = client.get("/stats")+ assert response.status_code == 200+ return response.json()["tokens"]["output_reduction"]+++def test_stats_publishes_the_estimate_while_steering_is_live(tmp_path, monkeypatch):+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")+ payload = _stats_output_reduction(tmp_path, monkeypatch)+ assert payload["active"] is True+ assert payload["method"] == "measured"+ assert payload["tokens_saved"] == 200+ assert payload["reduction_percent"] == 20.0+++def test_stats_gates_an_explicit_verbosity_zero(tmp_path, monkeypatch):+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "0")+ payload = _stats_output_reduction(tmp_path, monkeypatch)+ assert payload["active"] is False+ assert payload["method"] == "inactive"+ assert payload["tokens_saved"] == 0+ assert payload["reduction_percent"] == 0.0+++def test_stats_gates_a_learned_verbosity_zero(tmp_path, monkeypatch):+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)+ (tmp_path / "verbosity.json").write_text('{"verbosity_level": 0}')+ payload = _stats_output_reduction(tmp_path, monkeypatch)+ assert payload["active"] is False+ assert payload["method"] == "inactive"+++def test_stats_gates_a_controller_verbosity_zero(tmp_path, monkeypatch):+ monkeypatch.delenv("HEADROOM_VERBOSITY_LEVEL", raising=False)+ monkeypatch.setenv("HEADROOM_VERBOSITY_AUTOTUNE", "1")+ (tmp_path / "verbosity_controller.json").write_text('{"level": 0}')+ payload = _stats_output_reduction(tmp_path, monkeypatch)+ assert payload["active"] is False+ assert payload["method"] == "inactive"+++def test_stats_gates_cache_mode_even_with_a_level_set(tmp_path, monkeypatch):+ """``mode="cache"`` forces the resolved level to 0 in the handlers. /stats+ omitted ``steering_allowed_for``, so it kept publishing the ledger."""+ monkeypatch.setenv("HEADROOM_VERBOSITY_LEVEL", "3")+ payload = _stats_output_reduction(tmp_path, monkeypatch, mode="cache")+ assert payload["active"] is False+ assert payload["method"] == "inactive"