Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(savings): stop scoring unobserved strata against the global mean (#3128)
gglucassf8645257c074
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.pyindex 0b87c0039..9a46db439 100644--- a/headroom/proxy/output_savings.py+++ b/headroom/proxy/output_savings.py@@ -229,12 +229,23 @@ class BaselineModel:self.strata.setdefault(key, _Accum()).merge(acc)self.glob.merge(other.glob)- def lookup(self, key: str) -> tuple[float, float, int]:+ def lookup(self, key: str, *, fall_back_to_global: bool = False) -> tuple[float, float, int]:"""Return ``(mean, var, n)`` for *key* with hierarchical back-off.- Falls back by trimming trailing (least-specific) stratum fields, then- to the global mean. Back-off keeps the estimate defined for strata the- baseline never saw, at the cost of specificity.+ Falls back by trimming trailing (least-specific) stratum fields, so a+ stratum the baseline never saw is still scored against its nearest+ observed neighbours. Returns ``(0.0, 0.0, 0)`` when even that finds+ nothing -- callers already treat ``n == 0`` as "no evidence".++ ``fall_back_to_global`` restores the old last resort of the+ all-requests mean. It is off by default because that mean is not a+ control for anything: the baseline is seeded once, from whatever the+ user ran *before* installing, so every model family they adopt later+ resolves to it. On a real ledger that meant 48% of requests -- sonnet+ and fable turns whose replies average 43-770 tokens -- being scored+ against one opus-derived mean of 1,083, which alone produced 74% of the+ reported savings. Scoring a short no-tool ask against a long+ tool-calling turn is not a synthetic control, it is a unit conversion."""acc = self.strata.get(key)if acc is not None and acc.n > 0:@@ -242,11 +253,16 @@ class BaselineModel:parts = key.split("|")while len(parts) > 1:parts = parts[:-1]- prefix = "|".join(parts)+ prefix = "|".join(parts) + "|"+ neighbours = _Accum()for k, a in self.strata.items():- if k.startswith(prefix + "|") and a.n > 0:- return a.mean, a.var, a.n- return self.glob.mean, self.glob.var, self.glob.n+ if a.n > 0 and k.startswith(prefix):+ neighbours.merge(a)+ if neighbours.n > 0:+ return neighbours.mean, neighbours.var, neighbours.n+ if fall_back_to_global:+ return self.glob.mean, self.glob.var, self.glob.n+ return 0.0, 0.0, 0def to_dict(self) -> dict[str, Any]:return {diff --git a/tests/test_output_savings.py b/tests/test_output_savings.pyindex 6728f9c74..b9d53bab5 100644--- a/tests/test_output_savings.py+++ b/tests/test_output_savings.py@@ -189,14 +189,38 @@ class TestBaselineModel:assert mean == 500.0assert n == 1- def test_lookup_falls_back_to_global(self):+ def test_an_unobserved_family_is_not_scored_against_the_global_mean(self):+ # The baseline is seeded once, from what the user ran before installing.+ # Every family adopted later lands here, and the all-requests mean is+ # not a control for any of them.m = BaselineModel()m.observe("opus|a|s|tools", 100)m.observe("sonnet|b|m|notools", 300)- mean, _, n = m.lookup("gpt|totally|xl|tools")+ assert m.lookup("gpt|totally|xl|tools") == (0.0, 0.0, 0)++ def test_the_global_mean_remains_available_on_request(self):+ m = BaselineModel()+ m.observe("opus|a|s|tools", 100)+ m.observe("sonnet|b|m|notools", 300)+ mean, _, n = m.lookup("gpt|totally|xl|tools", fall_back_to_global=True)assert mean == 200.0 # global mean of 100 and 300assert n == 2+ def test_prefix_backoff_merges_every_neighbour_not_the_first_one_hashed(self):+ # Taking the first matching stratum in dict order made the answer+ # depend on insertion order, so a round-tripped ledger could score the+ # same request differently.+ m = BaselineModel()+ m.observe("opus|ask|l|tools", 1000)+ m.observe("opus|ask|l|notools", 100)+ mean, _, n = m.lookup("opus|ask|l|other")+ assert (mean, n) == (550.0, 2)++ reversed_order = BaselineModel()+ reversed_order.observe("opus|ask|l|notools", 100)+ reversed_order.observe("opus|ask|l|tools", 1000)+ assert reversed_order.lookup("opus|ask|l|other") == m.lookup("opus|ask|l|other")+def test_roundtrip_serialization(self):m = BaselineModel()for v in (10, 20, 30):@@ -288,6 +312,28 @@ class TestEstimateFromBaseline:# ---------------------------------------------------------------------------+class TestEstimateExcludesUnobservedStrata:+ def test_requests_without_baseline_evidence_are_left_out(self):+ ledger = SavingsLedger()+ for _ in range(10):+ ledger.baseline.observe("opus|ask|l|tools", 1000)+ ledger.record("treatment", "opus|ask|l|tools", 800)+ # A family the baseline never saw, with far shorter replies. Scoring it+ # against the global mean would credit ~950 saved tokens per request.+ for _ in range(40):+ ledger.record("treatment", "sonnet|new_user_ask|m|notools", 50)++ est = ledger.estimate_from_baseline()+ assert est.n_requests == 10, "only the observed stratum is scored"+ assert abs(est.tokens_saved - 2000) < 1e-6 # 10 * (1000 - 800)+ assert abs(est.pct - 20.0) < 1e-6++ def test_per_request_savings_are_zero_without_evidence(self):+ ledger = SavingsLedger()+ ledger.baseline.observe("opus|ask|l|tools", 1000)+ assert ledger.baseline.lookup("sonnet|new_user_ask|m|notools") == (0.0, 0.0, 0)++class TestEstimateFromHoldout:def test_none_without_control_data(self):ledger = SavingsLedger()