Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(savings): don't let a thin holdout arm decide the headline number (#3127)

gglucass2a12e035a323
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py
index 9a46db439..333c03d48 100644
--- a/headroom/proxy/output_savings.py
+++ b/headroom/proxy/output_savings.py
@@ -354,6 +354,44 @@ class SavingsEstimate:
_SHAPED_LABEL_PREFIX = "output_shaper:verbosity:"
+# :data:`MEASURED_MIN_CLUSTERS` decides whether a stratum is a real sample at
+# all. The gates below decide something stricter: whether the measurement has
+# earned the headline slot away from the synthetic control.
+
+# Conversations per arm a stratum needs before it may count toward the
+# headline. The 95% band is computed per REQUEST, so it reads one long
+# conversation as many independent draws: five control conversations of 200
+# requests each report about +/-4pp where the honest band is tens of points.
+# Only capped per-stratum cluster ids are stored, so a cluster-robust band
+# cannot be computed; this floor is the clustering guard instead. It must stay
+# at or below ``_CLUSTER_CAP``, which ``n_clusters`` saturates at.
+MEASURED_SUPERSEDE_MIN_CLUSTERS = 30
+
+# Share of the conversation-labelled treatment requests the measured strata
+# must cover. Below it the holdout describes a corner of the traffic, not the
+# traffic. Labelled requests only: the measurement can only ever be built from
+# those, so unlabelled legacy volume must neither help nor block it.
+MEASURED_MIN_COVERAGE = 0.5
+
+# ...and the 95% band must be at least this tight, in percentage points of
+# reduction. A stratum can clear every cluster floor and still be too noisy to
+# say anything.
+MEASURED_MAX_CI_HALF_WIDTH_PCT = 10.0
+
+
+def _measured_covers(measured: SavingsEstimate, labelled_requests: int) -> bool:
+ """Whether the measured strata describe the traffic, not a corner of it."""
+ return measured.n_requests >= MEASURED_MIN_COVERAGE * labelled_requests
+
+
+def _measured_supersedes(measured: SavingsEstimate, labelled_requests: int) -> bool:
+ """Whether the A/B measurement has outgrown the synthetic-control estimate."""
+ half_width = (measured.ci_high_pct - measured.ci_low_pct) / 2.0
+ if half_width > MEASURED_MAX_CI_HALF_WIDTH_PCT:
+ return False
+ return _measured_covers(measured, labelled_requests)
+
+
@dataclass
class SavingsLedger:
"""Accumulates shaped (treatment) and unshaped (control) observations and
@@ -407,7 +445,9 @@ class SavingsLedger:
var += (n * n) * (mu_var / m)
return self._finalize(total_saved, total_baseline, var, n_requests, "estimated")
- def estimate_from_holdout(self) -> SavingsEstimate | None:
+ def estimate_from_holdout(
+ self, min_clusters: int = MEASURED_MIN_CLUSTERS
+ ) -> SavingsEstimate | None:
"""A/B measurement: per-stratum control mean minus treatment mean.
Only strata with conversation-labelled data in BOTH arms contribute,
@@ -439,7 +479,7 @@ class SavingsLedger:
c = self.control.get(key)
if c is None or c.qn == 0 or t.qn == 0:
continue
- if c.n_clusters < MEASURED_MIN_CLUSTERS or t.n_clusters < MEASURED_MIN_CLUSTERS:
+ if c.n_clusters < min_clusters or t.n_clusters < min_clusters:
continue
contributing += 1
# Everything below reads the qualified subset only. The clusters
@@ -529,17 +569,33 @@ class SavingsLedger:
)
def best_estimate(self, level: int | None = None) -> SavingsEstimate:
- """Strongest available tier: measured > estimated > modelled.
+ """Strongest believable tier: measured > estimated > modelled.
``level`` enables the modelled fallback; without it the behaviour is
unchanged from before, which keeps every existing caller honest.
+
+ The measured tier is preferred only once it is worth believing. With
+ conversation-stable keys five conversations per arm fill quickly, so on
+ the bare cluster gate a corner of the traffic, or a few long
+ conversations, would decide the headline. The measured number displaces
+ the synthetic control only when every stratum it is built from holds
+ :data:`MEASURED_SUPERSEDE_MIN_CLUSTERS` conversations per arm, those
+ strata cover :data:`MEASURED_MIN_COVERAGE` of the labelled treatment
+ requests, and its band is tight enough to mean something. With no
+ baseline to fall back on, the bare cluster gate's measurement is still
+ reported, since it displaces nothing, but only while it covers the
+ traffic: a corner of it is not a headline either way.
"""
- measured = self.estimate_from_holdout()
- if measured is not None:
+ measured = self.estimate_from_holdout(min_clusters=MEASURED_SUPERSEDE_MIN_CLUSTERS)
+ labelled = sum(t.qn for t in self.treatment.values())
+ if measured is not None and _measured_supersedes(measured, labelled):
return measured
estimated = self.estimate_from_baseline()
if estimated.n_requests > 0:
return estimated
+ measured = self.estimate_from_holdout()
+ if measured is not None and _measured_covers(measured, labelled):
+ return measured
if level is not None:
modelled = self.estimate_from_model(level)
if modelled is not None:
diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py
index b9d53bab5..e8ad6d749 100644
--- a/tests/test_output_savings.py
+++ b/tests/test_output_savings.py
@@ -8,6 +8,7 @@ import pytest
from headroom.proxy.output_savings import (
MEASURED_MIN_CLUSTERS,
+ MEASURED_SUPERSEDE_MIN_CLUSTERS,
BaselineModel,
SavingsLedger,
SavingsRecorder,
@@ -365,7 +366,7 @@ class TestEstimateFromHoldout:
def test_best_estimate_prefers_measured(self):
ledger = SavingsLedger()
- for i in range(10):
+ for i in range(MEASURED_SUPERSEDE_MIN_CLUSTERS):
ledger.baseline.observe("opus|a|s|tools", 1000)
ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")
ledger.record("treatment", "opus|a|s|tools", 900, f"t{i}")
@@ -378,6 +379,91 @@ class TestEstimateFromHoldout:
ledger.record("treatment", "opus|a|s|tools", 900)
assert ledger.best_estimate().kind == "estimated"
+ def test_a_wide_band_does_not_take_over_even_at_full_size(self):
+ """The width gate still bites once the cluster gate is cleared."""
+ ledger = SavingsLedger()
+ for i in range(200):
+ ledger.baseline.observe("opus|a|s|tools", 1000)
+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")
+ for i in range(30):
+ ledger.record("control", "opus|a|s|tools", (200, 1000, 5000)[i % 3], f"c{i}")
+
+ measured = ledger.estimate_from_holdout()
+ assert measured is not None, "the arm is spread over enough conversations"
+ assert (measured.ci_high_pct - measured.ci_low_pct) / 2 > 10, "and too noisy to believe"
+ assert ledger.best_estimate().kind == "estimated"
+
+ def test_a_holdout_covering_a_corner_of_traffic_does_not_take_over(self):
+ ledger = SavingsLedger()
+ # One stratum measured cleanly, but it is a fraction of the traffic.
+ for i in range(30):
+ ledger.baseline.observe("opus|a|s|tools", 1000)
+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")
+ ledger.record("treatment", "opus|a|s|tools", 900, f"t{i}")
+ for i in range(200):
+ ledger.baseline.observe("opus|b|xl|tools", 4000)
+ ledger.record("treatment", "opus|b|xl|tools", 3000, f"b{i}")
+ assert ledger.best_estimate().kind == "estimated"
+
+ def test_a_few_long_conversations_do_not_take_the_headline(self):
+ """Five control conversations clear the cluster gate, and their
+ per-request band looks tight, but they are five draws, not 1,000."""
+ ledger = SavingsLedger()
+ for i in range(200):
+ ledger.baseline.observe("opus|a|s|tools", 1000)
+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")
+ for conv in range(5):
+ for j in range(200):
+ ledger.record("control", "opus|a|s|tools", 900 + 100 * conv + j % 3, f"c{conv}")
+
+ measured = ledger.estimate_from_holdout()
+ assert measured is not None, "a real arm by the cluster gate"
+ assert (measured.ci_high_pct - measured.ci_low_pct) / 2 < 10, "and a tight-looking band"
+ assert ledger.best_estimate().kind == "estimated"
+
+ def test_unlabelled_legacy_traffic_does_not_block_the_measurement(self):
+ """Requests recorded before conversations were tracked can never be
+ measured, so they must not count against the measurement's coverage."""
+ ledger = SavingsLedger()
+ for _ in range(2000):
+ ledger.baseline.observe("opus|a|s|tools", 1000)
+ ledger.record("treatment", "opus|a|s|tools", 800)
+ for i in range(MEASURED_SUPERSEDE_MIN_CLUSTERS):
+ ledger.record("control", "opus|a|s|tools", 1000 + i % 3, f"c{i}")
+ ledger.record("treatment", "opus|a|s|tools", 900 + i % 3, f"t{i}")
+
+ assert ledger.estimate_from_baseline().n_requests == 2030
+ assert ledger.best_estimate().kind == "measured"
+
+ def test_a_young_holdout_still_reports_without_a_baseline(self):
+ """Below the supersede floor there is nothing to displace, so the
+ cluster-gated measurement is reported rather than nothing."""
+ ledger = SavingsLedger()
+ for i in range(MEASURED_MIN_CLUSTERS):
+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")
+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")
+ assert ledger.best_estimate().kind == "measured"
+
+ def test_a_corner_holdout_does_not_set_the_headline_without_a_baseline(self):
+ ledger = SavingsLedger()
+ for i in range(MEASURED_MIN_CLUSTERS):
+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")
+ ledger.record("treatment", "opus|a|s|tools", 100, f"t{i}")
+ # The rest of the traffic has no control to compare against.
+ for i in range(200):
+ ledger.record("treatment", "opus|b|xl|tools", 3000, f"b{i}")
+ assert ledger.estimate_from_holdout() is not None
+ assert ledger.best_estimate().kind != "measured"
+
+ def test_measured_wins_without_a_baseline_to_fall_back_on(self):
+ """A holdout-only deployment (no ``learn --verbosity`` run) still reports."""
+ ledger = SavingsLedger()
+ for i in range(50):
+ ledger.record("control", "opus|a|s|tools", 1000, f"c{i}")
+ ledger.record("treatment", "opus|a|s|tools", 800, f"t{i}")
+ assert ledger.estimate_from_baseline().n_requests == 0
+ assert ledger.best_estimate().kind == "measured"
+
class TestHoldoutClusterGate:
"""A stratum needs distinct CONVERSATIONS in both arms, not requests.