Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(telemetry): stop mixing tokenizer scales in RequestOutcome, and fix the overhead framing (#2756)
Tejas Chopra04e1517ede0a
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 3f0adc904..ffe770ef5 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4015,10 +4015,15 @@ class OpenAIHandlerMixin:provider=self.anthropic_backend.name,model=model,original_tokens=original_tokens,- optimized_tokens=total_input_tokens,+ # Local count, same tokenizer as original_tokens, so+ # every delta built from the pair is coherent. The+ # provider's own number rides along separately for+ # billing — see RequestOutcome's field docs.+ optimized_tokens=optimized_tokens,+ provider_input_tokens=total_input_tokens,output_tokens=output_tokens,tokens_saved=tokens_saved,- attempted_input_tokens=total_input_tokens + tokens_saved,+ attempted_input_tokens=optimized_tokens + tokens_saved,cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,uncached_input_tokens=uncached_input_tokens,@@ -4425,10 +4430,12 @@ class OpenAIHandlerMixin:model=model,status_code=response.status_code,original_tokens=original_tokens,- optimized_tokens=total_input_tokens,+ # Same-tokenizer pair; provider count carried separately.+ optimized_tokens=optimized_tokens,+ provider_input_tokens=total_input_tokens,output_tokens=output_tokens,tokens_saved=tokens_saved,- attempted_input_tokens=total_input_tokens + tokens_saved,+ attempted_input_tokens=optimized_tokens + tokens_saved,cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,uncached_input_tokens=uncached_input_tokens,@@ -5417,13 +5424,15 @@ class OpenAIHandlerMixin:)uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)- effective_optimized_tokens = (- total_input_tokens if total_input_tokens > 0 else optimized_tokens- )- effective_original_tokens = max(- original_tokens,- effective_optimized_tokens + tokens_saved,- )+ # Was: optimized := provider count, then original := max(original,+ # optimized + saved) to stop `attempted` exceeding `original`.+ # That paper over the symptom by inflating `original` upward,+ # which is why the beacon still shipped `eligible_pct > 100`+ # from the other emit sites that lacked the same fudge. The+ # real cause was mixing tokenizer scales; keep the local pair+ # coherent and hand the provider count over separately.+ effective_optimized_tokens = optimized_tokens+ effective_original_tokens = original_tokens_resp_log_tags = {**(tags or {}),@@ -5446,6 +5455,7 @@ class OpenAIHandlerMixin:status_code=response.status_code,original_tokens=effective_original_tokens,optimized_tokens=effective_optimized_tokens,+ provider_input_tokens=total_input_tokens,output_tokens=output_tokens,tokens_saved=tokens_saved,attempted_input_tokens=attempted_input_tokens,diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex f25b99a7b..e0b304d4c 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -57,8 +57,24 @@ class RequestOutcome:# ── Tokens (required — every site has these) ──────────────────────# original_tokens: pre-compression request size, for `tok_before`- # optimized_tokens: post-compression bytes actually forwarded, for- # ``input_tokens`` and ``tok_after``+ # optimized_tokens: post-compression size actually forwarded, for+ # ``tok_after``. MUST be counted with the SAME tokenizer as+ # ``original_tokens`` — every derived quantity is a delta between the+ # two (``tokens_saved``, ``tokens_inflated``, ``attempted_input_tokens``,+ # and the beacon's ``eligible_pct`` / ``yield_pct``), so mixing scales+ # silently corrupts all of them. Handlers used to pass the provider's+ # ``usage.prompt_tokens`` here because it also fed billing; that made+ # ``tok_after`` a provider count against a locally-estimated+ # ``tok_before``. On a gpt-4o-mini turn where our estimator undercounted+ # by 2 tokens that shipped as ``eligible_pct: 120`` — a structurally+ # impossible ratio — plus a phantom ``tok_inflated``. Provider-reported+ # input now lives in ``provider_input_tokens``.+ # provider_input_tokens: the provider's own prompt-token count for this+ # request, when it reported one (0 otherwise). This is the billed+ # quantity, so cost and volume totals use it in preference to+ # ``optimized_tokens``. Kept separate precisely because it is on the+ # provider's tokenizer scale and must never be differenced against+ # ``original_tokens``.# output_tokens: response tokens from upstream# tokens_saved: original - optimized (or 0 if compression bypassed)# attempted_input_tokens: denominator for active-savings-percent.@@ -71,6 +87,10 @@ class RequestOutcome:output_tokens: inttokens_saved: intattempted_input_tokens: int+ # Optional so the 18 existing emit sites need no change: a handler that has+ # no provider count (or whose optimized_tokens is already provider-scaled)+ # leaves it 0 and billing falls back to optimized_tokens, exactly as before.+ provider_input_tokens: int = 0# ── Cache (provider-agnostic; unused fields stay 0) ───────────────# Anthropic populates all five (read + write + 5m + 1h + uncached).@@ -439,11 +459,21 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:# session summary / cost summary / all-layers total can surface the layer.tool_search_saved = tool_schema_saved_from_tags(outcome.tags or {})+ # Billed input volume. Prefer the provider's own count where it reported one+ # — that is what the invoice charges for, and it is the number cache math is+ # already expressed in. Falls back to our local ``optimized_tokens`` when the+ # provider stayed silent (streaming without usage, or a non-reporting+ # backend), which is the pre-split behaviour for every handler.+ #+ # Deliberately NOT used for any delta: differencing this against+ # ``original_tokens`` mixes tokenizer scales. See the field docs.+ billed_input_tokens = outcome.provider_input_tokens or outcome.optimized_tokens+# 1. Prometheus / SavingsTracker.await handler.metrics.record_request(provider=outcome.provider,model=outcome.model,- input_tokens=outcome.optimized_tokens,+ input_tokens=billed_input_tokens,output_tokens=outcome.output_tokens,tokens_saved=outcome.tokens_saved,latency_ms=outcome.total_latency_ms,@@ -462,6 +492,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:project=project,client=outcome.client,tool_search_saved=tool_search_saved,+ local_input_tokens=outcome.optimized_tokens,)# 2. Cost tracker (optional).@@ -470,7 +501,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:cost_tracker.record_tokens(outcome.model,outcome.tokens_saved,- outcome.optimized_tokens,+ billed_input_tokens,cache_read_tokens=outcome.cache_read_tokens,cache_write_tokens=outcome.cache_write_tokens,cache_write_5m_tokens=outcome.cache_write_5m_tokens,diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex e3f2d781a..4f691b79b 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -690,8 +690,17 @@ class PrometheusMetrics:project: str | None = None,client: str | None = None,tool_search_saved: int = 0,+ local_input_tokens: int | None = None,):- """Record metrics for a request."""+ """Record metrics for a request.++ ``input_tokens`` is the billed/volume figure and may be the provider's own+ count. ``local_input_tokens`` is the same request measured with the SAME+ tokenizer as ``tokens_saved``; it is used wherever a delta is derived, so+ reduction/yield/ledger math never straddles two rulers. Defaults to+ ``input_tokens`` when omitted, preserving pre-split behaviour.+ """+ ledger_input_tokens = input_tokens if local_input_tokens is None else local_input_tokens# Post-guard invariant (all providers): Headroom never forwards a request# larger than the original — handlers revert any inflation before sending# (verified clean on the wire). So compression savings are >= 0; a negative@@ -848,8 +857,14 @@ class PrometheusMetrics:# Reconstruct the original as forwarded + saved.await asyncio.to_thread(savings_ledger.record_savings_event,- tokens_before=input_tokens + tokens_saved,- tokens_after=input_tokens,+ # The ledger stores a DELTA, so both ends must be on one ruler.+ # `input_tokens` is the billed/volume figure and may be the+ # provider's own count; pairing it with a locally-counted+ # `tokens_saved` yields a mixed-ruler before/after (local 10->6+ # with the provider reporting 8 would record 12->8). Use the+ # caller's local count when supplied.+ tokens_before=ledger_input_tokens + tokens_saved,+ tokens_after=ledger_input_tokens,model=model,client=client or "proxy",source="proxy",diff --git a/headroom/telemetry/session.py b/headroom/telemetry/session.pyindex e7d9f0933..9049618a6 100644--- a/headroom/telemetry/session.py+++ b/headroom/telemetry/session.py@@ -328,13 +328,36 @@ class _Session:# Provider prompt cache participation. Headroom freezes prefixes# to protect this, so it is the other side of eligible_pct."cache_read_pct": _pct(self.cache_read_tokens, self.original_tokens),- # What fraction of wall-clock did Headroom itself add?+ # Headroom's share of REQUEST time: sum(overhead) / sum(latency),+ # i.e. a latency-weighted average across turns.+ #+ # NOT a fraction of wall-clock, which is what this comment used to+ # claim. Both terms are sums over turns, and turns run+ # concurrently (parallel tool calls, subagents, several clients on+ # one proxy), so each sum can exceed the session's elapsed time —+ # observed at 1.84x on a 1393-turn session. Dividing one+ # over-counted sum by another still yields a meaningful per-request+ # share, but it says nothing about how much longer the session took.+ # For that, compare `overhead_ms_per_turn` against+ # `session.duration_s / turns`."overhead_pct": _pct(self.overhead_ms, self.latency_ms),},"compression": {"transforms": dict(self.transforms),"overhead_ms_total": round(self.overhead_ms, 1),+ # Sum of per-request durations, NOT elapsed time: concurrent turns+ # make this exceed `session.duration_s`. Kept under the original+ # name for schema-v1 consumers; read the per-turn values below for+ # anything comparable across sessions."latency_ms_total": round(self.latency_ms, 1),+ # Unambiguous under concurrency: a mean per request, independent of+ # how many were in flight. This is the pair to reason about.+ "overhead_ms_per_turn": round(self.overhead_ms / self.turns, 1)+ if self.turns+ else 0.0,+ "latency_ms_per_turn": round(self.latency_ms / self.turns, 1)+ if self.turns+ else 0.0,"passthrough_turns": self.passthrough_turns,# Served from Headroom's own response cache — the provider was# never called at all. 100% saving on those turns.@@ -459,7 +482,12 @@ def _fold(sess: _Session, outcome: Any, now: float, source: str = "proxy") -> Nosess.sources[source] = sess.sources.get(source, 0) + 1sess.original_tokens += int(get("original_tokens") or 0)sess.attempted_tokens += int(get("attempted_input_tokens") or 0)- sess.input_tokens += int(get("optimized_tokens") or 0)+ # Billed/volume figure, so prefer the provider's own count and fall back to+ # the local one. It sits beside output/cache_read/cache_write/uncached, which+ # are all provider-reported, so making it local would put one local number in+ # a dict of provider numbers — and `tokens.input` is what a reader sums the+ # cache buckets against.+ sess.input_tokens += int(get("provider_input_tokens") or 0) or int(get("optimized_tokens") or 0)sess.output_tokens += int(get("output_tokens") or 0)sess.tokens_saved += int(get("tokens_saved") or 0)sess.cache_read_tokens += int(get("cache_read_tokens") or 0)diff --git a/tests/test_outcome_dual_ruler_funnel.py b/tests/test_outcome_dual_ruler_funnel.pynew file mode 100644index 000000000..a9db82e1f--- /dev/null+++ b/tests/test_outcome_dual_ruler_funnel.py@@ -0,0 +1,154 @@+"""One mismatched provider count must reach billing and delta math differently.++Splitting ``optimized_tokens`` (local) from ``provider_input_tokens`` (provider)+is only half the fix. Two consumers derive their own numbers downstream, and each+needs the OTHER side of the split:++* ``telemetry.session._fold`` accumulates ``tokens.input``, a billed/volume figure+ that sits beside ``output``/``cache_read``/``cache_write``/``uncached`` — all+ provider-reported. It must prefer the provider count.+* ``PrometheusMetrics.record_request`` reconstructs the durable savings ledger as+ ``tokens_before = input_tokens + tokens_saved``, ``tokens_after = input_tokens``.+ That is a DELTA, so pairing a provider ``input_tokens`` with a locally-counted+ ``tokens_saved`` straddles two rulers — local 10->6 with the provider reporting+ 8 would record 12->8.++These drive the real funnel with one deliberately mismatched pair and assert both+semantics, rather than asserting on the dataclass alone.+"""++from __future__ import annotations++import pytest++from headroom.proxy.outcome import RequestOutcome++# Local 10 -> 6 (saved 4); the provider says the prompt was 8. Every number below+# is derived from exactly this one mismatch.+_LOCAL_ORIGINAL = 10+_LOCAL_OPTIMIZED = 6+_LOCAL_SAVED = 4+_PROVIDER_INPUT = 8+++def _outcome() -> RequestOutcome:+ return RequestOutcome(+ request_id="r1",+ provider="openai",+ model="gpt-4o-mini",+ original_tokens=_LOCAL_ORIGINAL,+ optimized_tokens=_LOCAL_OPTIMIZED,+ provider_input_tokens=_PROVIDER_INPUT,+ output_tokens=5,+ tokens_saved=_LOCAL_SAVED,+ attempted_input_tokens=_LOCAL_OPTIMIZED + _LOCAL_SAVED,+ )+++def test_beacon_input_is_the_billed_provider_count() -> None:+ """``tokens.input`` is a volume figure and must not silently become local."""+ from headroom.telemetry import session as sess_mod++ sess = sess_mod._Session(sid="s1", started=0.0, last_seen=0.0) # type: ignore[attr-defined]+ sess_mod._fold(sess, _outcome(), now=0.0, source="proxy") # type: ignore[attr-defined]++ assert sess.input_tokens == _PROVIDER_INPUT, (+ "the beacon's input volume must use the provider count, not the local one"+ )+ # The local pair still drives the reduction ratios.+ assert sess.original_tokens == _LOCAL_ORIGINAL+ assert sess.tokens_saved == _LOCAL_SAVED+++def test_beacon_falls_back_to_local_when_no_provider_count() -> None:+ """Providers that report no usage must behave exactly as before the split."""+ from headroom.telemetry import session as sess_mod++ o = RequestOutcome(+ request_id="r2",+ provider="anthropic",+ model="claude-sonnet-4-6",+ original_tokens=_LOCAL_ORIGINAL,+ optimized_tokens=_LOCAL_OPTIMIZED,+ output_tokens=5,+ tokens_saved=_LOCAL_SAVED,+ attempted_input_tokens=_LOCAL_OPTIMIZED + _LOCAL_SAVED,+ )+ sess = sess_mod._Session(sid="s2", started=0.0, last_seen=0.0) # type: ignore[attr-defined]+ sess_mod._fold(sess, o, now=0.0, source="proxy") # type: ignore[attr-defined]++ assert sess.input_tokens == _LOCAL_OPTIMIZED+++async def test_ledger_delta_stays_on_the_local_ruler(monkeypatch) -> None:+ """Drive the real record_request and capture what reaches the ledger.++ The ledger stores a delta, so both ends must be local. With the billed input+ (8, provider) paired against a local tokens_saved (4), the old shape recorded+ 12 -> 8 for a request that actually went 10 -> 6.+ """+ from headroom.proxy import prometheus_metrics as pm++ seen: dict = {}++ def fake_record_savings_event(**kw):+ seen.update(kw)++ monkeypatch.setattr(pm.savings_ledger, "record_savings_event", fake_record_savings_event)++ metrics = pm.PrometheusMetrics()+ metrics._stateless = False # the ledger write is skipped when stateless++ await metrics.record_request(+ provider="openai",+ model="gpt-4o-mini",+ input_tokens=_PROVIDER_INPUT, # billed/volume: provider's count+ local_input_tokens=_LOCAL_OPTIMIZED, # same ruler as tokens_saved+ output_tokens=5,+ tokens_saved=_LOCAL_SAVED,+ latency_ms=1.0,+ )++ assert seen, "no ledger event recorded"+ assert seen["tokens_before"] == _LOCAL_ORIGINAL, "before must be the local original"+ assert seen["tokens_after"] == _LOCAL_OPTIMIZED, "after must be the local optimized"+ # The mixed-ruler shape this replaces.+ assert (seen["tokens_before"], seen["tokens_after"]) != (+ _PROVIDER_INPUT + _LOCAL_SAVED,+ _PROVIDER_INPUT,+ )+ # Volume still counted on the billed figure.+ assert metrics.tokens_input_total == _PROVIDER_INPUT+++async def test_ledger_falls_back_to_billed_when_local_omitted(monkeypatch) -> None:+ """Pre-split callers keep their existing (single-value) behaviour."""+ from headroom.proxy import prometheus_metrics as pm++ seen: dict = {}+ monkeypatch.setattr(pm.savings_ledger, "record_savings_event", lambda **kw: seen.update(kw))+ metrics = pm.PrometheusMetrics()+ metrics._stateless = False++ await metrics.record_request(+ provider="openai",+ model="gpt-4o-mini",+ input_tokens=_PROVIDER_INPUT,+ output_tokens=5,+ tokens_saved=_LOCAL_SAVED,+ latency_ms=1.0,+ )++ assert seen["tokens_after"] == _PROVIDER_INPUT+++def test_record_request_defaults_local_to_billed_when_omitted() -> None:+ """Callers that never pass local_input_tokens keep pre-split behaviour."""+ import inspect++ from headroom.proxy.prometheus_metrics import PrometheusMetrics++ sig = inspect.signature(PrometheusMetrics.record_request)+ assert sig.parameters["local_input_tokens"].default is Nonediff --git a/tests/test_outcome_token_scale.py b/tests/test_outcome_token_scale.pynew file mode 100644index 000000000..c6153bc28--- /dev/null+++ b/tests/test_outcome_token_scale.py@@ -0,0 +1,97 @@+"""``original_tokens`` and ``optimized_tokens`` must share a tokenizer scale.++Every derived quantity is a delta between the two — ``tokens_saved``,+``tokens_inflated``, ``attempted_input_tokens``, and the beacon's+``eligible_pct`` / ``yield_pct``. Handlers used to pass the provider's+``usage.prompt_tokens`` as ``optimized_tokens`` because it also fed billing,+which put a provider count against a locally-estimated ``original_tokens``.++A real beacon payload from a gpt-4o-mini session, where our estimator+undercounted by 2 tokens on a 10-token request:++ "tokens": {"original": 10, "attempted": 12, "input": 12, "saved": 0}+ "rates": {"eligible_pct": 120}++120% is structurally impossible — you cannot attempt to compress more than+arrived. The same mismatch also produces a phantom ``tok_inflated``.+"""++from __future__ import annotations++from headroom.proxy.outcome import RequestOutcome++# The exact numbers from the reported beacon entry.+_LOCAL_ORIGINAL = 10+_LOCAL_OPTIMIZED = 10 # nothing compressed: a 10-token request is below every floor+_PROVIDER_COUNT = 12 # gpt-4o-mini's own prompt_tokens — a different ruler+++def _outcome(**kw) -> RequestOutcome:+ base: dict = {+ "request_id": "r1",+ "provider": "openai",+ "model": "gpt-4o-mini",+ "original_tokens": _LOCAL_ORIGINAL,+ "optimized_tokens": _LOCAL_OPTIMIZED,+ "output_tokens": 5,+ "tokens_saved": 0,+ "attempted_input_tokens": _LOCAL_OPTIMIZED,+ }+ base.update(kw)+ return RequestOutcome(**base)+++def test_provider_count_does_not_contaminate_the_local_pair() -> None:+ """The provider's number rides alongside instead of replacing tok_after."""+ o = _outcome(provider_input_tokens=_PROVIDER_COUNT)++ assert o.original_tokens == _LOCAL_ORIGINAL+ assert o.optimized_tokens == _LOCAL_OPTIMIZED+ assert o.provider_input_tokens == _PROVIDER_COUNT+++def test_no_phantom_inflation_when_the_provider_counts_higher() -> None:+ """The regression: provider 12 vs local 10 reported 2 tokens of growth."""+ fixed = _outcome(provider_input_tokens=_PROVIDER_COUNT)+ assert fixed.tokens_inflated == 0++ # What the old shape produced — optimized carrying the provider count.+ contaminated = _outcome(optimized_tokens=_PROVIDER_COUNT)+ assert contaminated.tokens_inflated == 2, (+ "guard is inverted; this asserts the OLD behaviour to document the bug"+ )+++def test_attempted_cannot_exceed_original_on_a_no_op_turn() -> None:+ """`eligible_pct = attempted / original` must stay <= 100 here.++ attempted is built as optimized + saved, so once optimized is local the+ ratio is bounded by original for any turn that did not grow.+ """+ o = _outcome(+ provider_input_tokens=_PROVIDER_COUNT,+ attempted_input_tokens=_LOCAL_OPTIMIZED + 0,+ )+ assert o.attempted_input_tokens <= o.original_tokens+ assert 100 * o.attempted_input_tokens / o.original_tokens == 100.0+++def test_provider_count_is_optional_and_defaults_to_zero() -> None:+ """18 pre-existing emit sites pass nothing; billing must fall back."""+ o = _outcome()+ assert o.provider_input_tokens == 0+++def test_real_inflation_is_still_reported() -> None:+ """Fixing the scale must not mute genuine post-compression growth.++ Same tokenizer both sides, request forwarded larger (memory injection /+ proactive expansion) — that is real and must survive.+ """+ o = _outcome(+ original_tokens=55_161,+ optimized_tokens=57_845,+ provider_input_tokens=58_000,+ tokens_saved=0,+ )+ assert o.tokens_inflated == 2_684