Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(savings): don't let a thin holdout arm decide the headline number (#3127)
gglucass2a12e035a323
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.pyindex 9a46db439..333c03d48 100644--- a/headroom/proxy/output_savings.py+++ b/headroom/proxy/output_savings.py@@ -354,6 +354,44 @@ class SavingsEstimate:_SHAPED_LABEL_PREFIX = "output_shaper:verbosity:"+# :data:`MEASURED_MIN_CLUSTERS` decides whether a stratum is a real sample at+# all. The gates below decide something stricter: whether the measurement has+# earned the headline slot away from the synthetic control.++# Conversations per arm a stratum needs before it may count toward the+# headline. The 95% band is computed per REQUEST, so it reads one long+# conversation as many independent draws: five control conversations of 200+# requests each report about +/-4pp where the honest band is tens of points.+# Only capped per-stratum cluster ids are stored, so a cluster-robust band+# cannot be computed; this floor is the clustering guard instead. It must stay+# at or below ``_CLUSTER_CAP``, which ``n_clusters`` saturates at.+MEASURED_SUPERSEDE_MIN_CLUSTERS = 30++# Share of the conversation-labelled treatment requests the measured strata+# must cover. Below it the holdout describes a corner of the traffic, not the+# traffic. Labelled requests only: the measurement can only ever be built from+# those, so unlabelled legacy volume must neither help nor block it.+MEASURED_MIN_COVERAGE = 0.5++# ...and the 95% band must be at least this tight, in percentage points of+# reduction. A stratum can clear every cluster floor and still be too noisy to+# say anything.+MEASURED_MAX_CI_HALF_WIDTH_PCT = 10.0+++def _measured_covers(measured: SavingsEstimate, labelled_requests: int) -> bool:+ """Whether the measured strata describe the traffic, not a corner of it."""+ return measured.n_requests >= MEASURED_MIN_COVERAGE * labelled_requests+++def _measured_supersedes(measured: SavingsEstimate, labelled_requests: int) -> bool:+ """Whether the A/B measurement has outgrown the synthetic-control estimate."""+ half_width = (measured.ci_high_pct - measured.ci_low_pct) / 2.0+ if half_width > MEASURED_MAX_CI_HALF_WIDTH_PCT:+ return False+ return _measured_covers(measured, labelled_requests)++@dataclassclass SavingsLedger:"""Accumulates shaped (treatment) and unshaped (control) observations and@@ -407,7 +445,9 @@ class SavingsLedger:var += (n * n) * (mu_var / m)return self._finalize(total_saved, total_baseline, var, n_requests, "estimated")- def estimate_from_holdout(self) -> SavingsEstimate | None:+ def estimate_from_holdout(+ self, min_clusters: int = MEASURED_MIN_CLUSTERS+ ) -> SavingsEstimate | None:"""A/B measurement: per-stratum control mean minus treatment mean.Only strata with conversation-labelled data in BOTH arms contribute,@@ -439,7 +479,7 @@ class SavingsLedger:c = self.control.get(key)if c is None or c.qn == 0 or t.qn == 0:continue- if c.n_clusters < MEASURED_MIN_CLUSTERS or t.n_clusters < MEASURED_MIN_CLUSTERS:+ if c.n_clusters < min_clusters or t.n_clusters < min_clusters:continuecontributing += 1# Everything below reads the qualified subset only. The clusters@@ -529,17 +569,33 @@ class SavingsLedger:)def best_estimate(self, level: int | None = None) -> SavingsEstimate:- """Strongest available tier: measured > estimated > modelled.+ """Strongest believable tier: measured > estimated > modelled.``level`` enables the modelled fallback; without it the behaviour isunchanged from before, which keeps every existing caller honest.++ The measured tier is preferred only once it is worth believing. With+ conversation-stable keys five conversations per arm fill quickly, so on+ the bare cluster gate a corner of the traffic, or a few long+ conversations, would decide the headline. The measured number displaces+ the synthetic control only when every stratum it is built from holds+ :data:`MEASURED_SUPERSEDE_MIN_CLUSTERS` conversations per arm, those+ strata cover :data:`MEASURED_MIN_COVERAGE` of the labelled treatment+ requests, and its band is tight enough to mean something. With no+ baseline to fall back on, the bare cluster gate's measurement is still+ reported, since it displaces nothing, but only while it covers the+ traffic: a corner of it is not a headline either way."""- measured = self.estimate_from_holdout()- if measured is not None:+ measured = self.estimate_from_holdout(min_clusters=MEASURED_SUPERSEDE_MIN_CLUSTERS)+ labelled = sum(t.qn for t in self.treatment.values())+ if measured is not None and _measured_supersedes(measured, labelled):return measuredestimated = self.estimate_from_baseline()if estimated.n_requests > 0:return estimated+ measured = self.estimate_from_holdout()+ if measured is not None and _measured_covers(measured, labelled):+ return measuredif level is not None:modelled = self.estimate_from_model(level)if modelled is not None:diff --git a/tests/test_output_savings.py b/tests/test_output_savings.pyindex b9d53bab5..e8ad6d749 100644--- a/tests/test_output_savings.py+++ b/tests/test_output_savings.py@@ -8,6 +8,7 @@ import pytestfrom headroom.proxy.output_savings import (MEASURED_MIN_CLUSTERS,+ MEASURED_SUPERSEDE_MIN_CLUSTERS,BaselineModel,SavingsLedger,SavingsRecorder,@@ -365,7 +366,7 @@ class TestEstimateFromHoldout:def test_best_estimate_prefers_measured(self):ledger = SavingsLedger()- for i in range(10):+ for i in range(MEASURED_SUPERSEDE_MIN_CLUSTERS):ledger.baseline.observe("opus|a|s|tools", 1000)ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")ledger.record("treatment", "opus|a|s|tools", 900, f"t{i}")@@ -378,6 +379,91 @@ class TestEstimateFromHoldout:ledger.record("treatment", "opus|a|s|tools", 900)assert ledger.best_estimate().kind == "estimated"+ def test_a_wide_band_does_not_take_over_even_at_full_size(self):+ """The width gate still bites once the cluster gate is cleared."""+ ledger = SavingsLedger()+ for i in range(200):+ ledger.baseline.observe("opus|a|s|tools", 1000)+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")+ for i in range(30):+ ledger.record("control", "opus|a|s|tools", (200, 1000, 5000)[i % 3], f"c{i}")++ measured = ledger.estimate_from_holdout()+ assert measured is not None, "the arm is spread over enough conversations"+ assert (measured.ci_high_pct - measured.ci_low_pct) / 2 > 10, "and too noisy to believe"+ assert ledger.best_estimate().kind == "estimated"++ def test_a_holdout_covering_a_corner_of_traffic_does_not_take_over(self):+ ledger = SavingsLedger()+ # One stratum measured cleanly, but it is a fraction of the traffic.+ for i in range(30):+ ledger.baseline.observe("opus|a|s|tools", 1000)+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")+ ledger.record("treatment", "opus|a|s|tools", 900, f"t{i}")+ for i in range(200):+ ledger.baseline.observe("opus|b|xl|tools", 4000)+ ledger.record("treatment", "opus|b|xl|tools", 3000, f"b{i}")+ assert ledger.best_estimate().kind == "estimated"++ def test_a_few_long_conversations_do_not_take_the_headline(self):+ """Five control conversations clear the cluster gate, and their+ per-request band looks tight, but they are five draws, not 1,000."""+ ledger = SavingsLedger()+ for i in range(200):+ ledger.baseline.observe("opus|a|s|tools", 1000)+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")+ for conv in range(5):+ for j in range(200):+ ledger.record("control", "opus|a|s|tools", 900 + 100 * conv + j % 3, f"c{conv}")++ measured = ledger.estimate_from_holdout()+ assert measured is not None, "a real arm by the cluster gate"+ assert (measured.ci_high_pct - measured.ci_low_pct) / 2 < 10, "and a tight-looking band"+ assert ledger.best_estimate().kind == "estimated"++ def test_unlabelled_legacy_traffic_does_not_block_the_measurement(self):+ """Requests recorded before conversations were tracked can never be+ measured, so they must not count against the measurement's coverage."""+ ledger = SavingsLedger()+ for _ in range(2000):+ ledger.baseline.observe("opus|a|s|tools", 1000)+ ledger.record("treatment", "opus|a|s|tools", 800)+ for i in range(MEASURED_SUPERSEDE_MIN_CLUSTERS):+ ledger.record("control", "opus|a|s|tools", 1000 + i % 3, f"c{i}")+ ledger.record("treatment", "opus|a|s|tools", 900 + i % 3, f"t{i}")++ assert ledger.estimate_from_baseline().n_requests == 2030+ assert ledger.best_estimate().kind == "measured"++ def test_a_young_holdout_still_reports_without_a_baseline(self):+ """Below the supersede floor there is nothing to displace, so the+ cluster-gated measurement is reported rather than nothing."""+ ledger = SavingsLedger()+ for i in range(MEASURED_MIN_CLUSTERS):+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")+ assert ledger.best_estimate().kind == "measured"++ def test_a_corner_holdout_does_not_set_the_headline_without_a_baseline(self):+ ledger = SavingsLedger()+ for i in range(MEASURED_MIN_CLUSTERS):+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")+ ledger.record("treatment", "opus|a|s|tools", 100, f"t{i}")+ # The rest of the traffic has no control to compare against.+ for i in range(200):+ ledger.record("treatment", "opus|b|xl|tools", 3000, f"b{i}")+ assert ledger.estimate_from_holdout() is not None+ assert ledger.best_estimate().kind != "measured"++ def test_measured_wins_without_a_baseline_to_fall_back_on(self):+ """A holdout-only deployment (no ``learn --verbosity`` run) still reports."""+ ledger = SavingsLedger()+ for i in range(50):+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")+ assert ledger.estimate_from_baseline().n_requests == 0+ assert ledger.best_estimate().kind == "measured"+class TestHoldoutClusterGate:"""A stratum needs distinct CONVERSATIONS in both arms, not requests.