Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(telemetry): stop mixing tokenizer scales in RequestOutcome, and fix the overhead framing (#2756)

Tejas Chopra04e1517ede0a
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 3f0adc904..ffe770ef5 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4015,10 +4015,15 @@ class OpenAIHandlerMixin:
provider=self.anthropic_backend.name,
model=model,
original_tokens=original_tokens,
- optimized_tokens=total_input_tokens,
+ # Local count, same tokenizer as original_tokens, so
+ # every delta built from the pair is coherent. The
+ # provider's own number rides along separately for
+ # billing — see RequestOutcome's field docs.
+ optimized_tokens=optimized_tokens,
+ provider_input_tokens=total_input_tokens,
output_tokens=output_tokens,
tokens_saved=tokens_saved,
- attempted_input_tokens=total_input_tokens + tokens_saved,
+ attempted_input_tokens=optimized_tokens + tokens_saved,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
uncached_input_tokens=uncached_input_tokens,
@@ -4425,10 +4430,12 @@ class OpenAIHandlerMixin:
model=model,
status_code=response.status_code,
original_tokens=original_tokens,
- optimized_tokens=total_input_tokens,
+ # Same-tokenizer pair; provider count carried separately.
+ optimized_tokens=optimized_tokens,
+ provider_input_tokens=total_input_tokens,
output_tokens=output_tokens,
tokens_saved=tokens_saved,
- attempted_input_tokens=total_input_tokens + tokens_saved,
+ attempted_input_tokens=optimized_tokens + tokens_saved,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
uncached_input_tokens=uncached_input_tokens,
@@ -5417,13 +5424,15 @@ class OpenAIHandlerMixin:
)
uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
- effective_optimized_tokens = (
- total_input_tokens if total_input_tokens > 0 else optimized_tokens
- )
- effective_original_tokens = max(
- original_tokens,
- effective_optimized_tokens + tokens_saved,
- )
+ # Was: optimized := provider count, then original := max(original,
+ # optimized + saved) to stop `attempted` exceeding `original`.
+ # That paper over the symptom by inflating `original` upward,
+ # which is why the beacon still shipped `eligible_pct > 100`
+ # from the other emit sites that lacked the same fudge. The
+ # real cause was mixing tokenizer scales; keep the local pair
+ # coherent and hand the provider count over separately.
+ effective_optimized_tokens = optimized_tokens
+ effective_original_tokens = original_tokens
_resp_log_tags = {
**(tags or {}),
@@ -5446,6 +5455,7 @@ class OpenAIHandlerMixin:
status_code=response.status_code,
original_tokens=effective_original_tokens,
optimized_tokens=effective_optimized_tokens,
+ provider_input_tokens=total_input_tokens,
output_tokens=output_tokens,
tokens_saved=tokens_saved,
attempted_input_tokens=attempted_input_tokens,
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index f25b99a7b..e0b304d4c 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -57,8 +57,24 @@ class RequestOutcome:
# ── Tokens (required — every site has these) ──────────────────────
# original_tokens: pre-compression request size, for `tok_before`
- # optimized_tokens: post-compression bytes actually forwarded, for
- # ``input_tokens`` and ``tok_after``
+ # optimized_tokens: post-compression size actually forwarded, for
+ # ``tok_after``. MUST be counted with the SAME tokenizer as
+ # ``original_tokens`` — every derived quantity is a delta between the
+ # two (``tokens_saved``, ``tokens_inflated``, ``attempted_input_tokens``,
+ # and the beacon's ``eligible_pct`` / ``yield_pct``), so mixing scales
+ # silently corrupts all of them. Handlers used to pass the provider's
+ # ``usage.prompt_tokens`` here because it also fed billing; that made
+ # ``tok_after`` a provider count against a locally-estimated
+ # ``tok_before``. On a gpt-4o-mini turn where our estimator undercounted
+ # by 2 tokens that shipped as ``eligible_pct: 120`` — a structurally
+ # impossible ratio — plus a phantom ``tok_inflated``. Provider-reported
+ # input now lives in ``provider_input_tokens``.
+ # provider_input_tokens: the provider's own prompt-token count for this
+ # request, when it reported one (0 otherwise). This is the billed
+ # quantity, so cost and volume totals use it in preference to
+ # ``optimized_tokens``. Kept separate precisely because it is on the
+ # provider's tokenizer scale and must never be differenced against
+ # ``original_tokens``.
# output_tokens: response tokens from upstream
# tokens_saved: original - optimized (or 0 if compression bypassed)
# attempted_input_tokens: denominator for active-savings-percent.
@@ -71,6 +87,10 @@ class RequestOutcome:
output_tokens: int
tokens_saved: int
attempted_input_tokens: int
+ # Optional so the 18 existing emit sites need no change: a handler that has
+ # no provider count (or whose optimized_tokens is already provider-scaled)
+ # leaves it 0 and billing falls back to optimized_tokens, exactly as before.
+ provider_input_tokens: int = 0
# ── Cache (provider-agnostic; unused fields stay 0) ───────────────
# Anthropic populates all five (read + write + 5m + 1h + uncached).
@@ -439,11 +459,21 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
# session summary / cost summary / all-layers total can surface the layer.
tool_search_saved = tool_schema_saved_from_tags(outcome.tags or {})
+ # Billed input volume. Prefer the provider's own count where it reported one
+ # — that is what the invoice charges for, and it is the number cache math is
+ # already expressed in. Falls back to our local ``optimized_tokens`` when the
+ # provider stayed silent (streaming without usage, or a non-reporting
+ # backend), which is the pre-split behaviour for every handler.
+ #
+ # Deliberately NOT used for any delta: differencing this against
+ # ``original_tokens`` mixes tokenizer scales. See the field docs.
+ billed_input_tokens = outcome.provider_input_tokens or outcome.optimized_tokens
+
# 1. Prometheus / SavingsTracker.
await handler.metrics.record_request(
provider=outcome.provider,
model=outcome.model,
- input_tokens=outcome.optimized_tokens,
+ input_tokens=billed_input_tokens,
output_tokens=outcome.output_tokens,
tokens_saved=outcome.tokens_saved,
latency_ms=outcome.total_latency_ms,
@@ -462,6 +492,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
project=project,
client=outcome.client,
tool_search_saved=tool_search_saved,
+ local_input_tokens=outcome.optimized_tokens,
)
# 2. Cost tracker (optional).
@@ -470,7 +501,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
cost_tracker.record_tokens(
outcome.model,
outcome.tokens_saved,
- outcome.optimized_tokens,
+ billed_input_tokens,
cache_read_tokens=outcome.cache_read_tokens,
cache_write_tokens=outcome.cache_write_tokens,
cache_write_5m_tokens=outcome.cache_write_5m_tokens,
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index e3f2d781a..4f691b79b 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -690,8 +690,17 @@ class PrometheusMetrics:
project: str | None = None,
client: str | None = None,
tool_search_saved: int = 0,
+ local_input_tokens: int | None = None,
):
- """Record metrics for a request."""
+ """Record metrics for a request.
+
+ ``input_tokens`` is the billed/volume figure and may be the provider's own
+ count. ``local_input_tokens`` is the same request measured with the SAME
+ tokenizer as ``tokens_saved``; it is used wherever a delta is derived, so
+ reduction/yield/ledger math never straddles two rulers. Defaults to
+ ``input_tokens`` when omitted, preserving pre-split behaviour.
+ """
+ ledger_input_tokens = input_tokens if local_input_tokens is None else local_input_tokens
# Post-guard invariant (all providers): Headroom never forwards a request
# larger than the original — handlers revert any inflation before sending
# (verified clean on the wire). So compression savings are >= 0; a negative
@@ -848,8 +857,14 @@ class PrometheusMetrics:
# Reconstruct the original as forwarded + saved.
await asyncio.to_thread(
savings_ledger.record_savings_event,
- tokens_before=input_tokens + tokens_saved,
- tokens_after=input_tokens,
+ # The ledger stores a DELTA, so both ends must be on one ruler.
+ # `input_tokens` is the billed/volume figure and may be the
+ # provider's own count; pairing it with a locally-counted
+ # `tokens_saved` yields a mixed-ruler before/after (local 10->6
+ # with the provider reporting 8 would record 12->8). Use the
+ # caller's local count when supplied.
+ tokens_before=ledger_input_tokens + tokens_saved,
+ tokens_after=ledger_input_tokens,
model=model,
client=client or "proxy",
source="proxy",
diff --git a/headroom/telemetry/session.py b/headroom/telemetry/session.py
index e7d9f0933..9049618a6 100644
--- a/headroom/telemetry/session.py
+++ b/headroom/telemetry/session.py
@@ -328,13 +328,36 @@ class _Session:
# Provider prompt cache participation. Headroom freezes prefixes
# to protect this, so it is the other side of eligible_pct.
"cache_read_pct": _pct(self.cache_read_tokens, self.original_tokens),
- # What fraction of wall-clock did Headroom itself add?
+ # Headroom's share of REQUEST time: sum(overhead) / sum(latency),
+ # i.e. a latency-weighted average across turns.
+ #
+ # NOT a fraction of wall-clock, which is what this comment used to
+ # claim. Both terms are sums over turns, and turns run
+ # concurrently (parallel tool calls, subagents, several clients on
+ # one proxy), so each sum can exceed the session's elapsed time —
+ # observed at 1.84x on a 1393-turn session. Dividing one
+ # over-counted sum by another still yields a meaningful per-request
+ # share, but it says nothing about how much longer the session took.
+ # For that, compare `overhead_ms_per_turn` against
+ # `session.duration_s / turns`.
"overhead_pct": _pct(self.overhead_ms, self.latency_ms),
},
"compression": {
"transforms": dict(self.transforms),
"overhead_ms_total": round(self.overhead_ms, 1),
+ # Sum of per-request durations, NOT elapsed time: concurrent turns
+ # make this exceed `session.duration_s`. Kept under the original
+ # name for schema-v1 consumers; read the per-turn values below for
+ # anything comparable across sessions.
"latency_ms_total": round(self.latency_ms, 1),
+ # Unambiguous under concurrency: a mean per request, independent of
+ # how many were in flight. This is the pair to reason about.
+ "overhead_ms_per_turn": round(self.overhead_ms / self.turns, 1)
+ if self.turns
+ else 0.0,
+ "latency_ms_per_turn": round(self.latency_ms / self.turns, 1)
+ if self.turns
+ else 0.0,
"passthrough_turns": self.passthrough_turns,
# Served from Headroom's own response cache — the provider was
# never called at all. 100% saving on those turns.
@@ -459,7 +482,12 @@ def _fold(sess: _Session, outcome: Any, now: float, source: str = "proxy") -> No
sess.sources[source] = sess.sources.get(source, 0) + 1
sess.original_tokens += int(get("original_tokens") or 0)
sess.attempted_tokens += int(get("attempted_input_tokens") or 0)
- sess.input_tokens += int(get("optimized_tokens") or 0)
+ # Billed/volume figure, so prefer the provider's own count and fall back to
+ # the local one. It sits beside output/cache_read/cache_write/uncached, which
+ # are all provider-reported, so making it local would put one local number in
+ # a dict of provider numbers — and `tokens.input` is what a reader sums the
+ # cache buckets against.
+ sess.input_tokens += int(get("provider_input_tokens") or 0) or int(get("optimized_tokens") or 0)
sess.output_tokens += int(get("output_tokens") or 0)
sess.tokens_saved += int(get("tokens_saved") or 0)
sess.cache_read_tokens += int(get("cache_read_tokens") or 0)
diff --git a/tests/test_outcome_dual_ruler_funnel.py b/tests/test_outcome_dual_ruler_funnel.py
new file mode 100644
index 000000000..a9db82e1f
--- /dev/null
+++ b/tests/test_outcome_dual_ruler_funnel.py
@@ -0,0 +1,154 @@
+"""One mismatched provider count must reach billing and delta math differently.
+
+Splitting ``optimized_tokens`` (local) from ``provider_input_tokens`` (provider)
+is only half the fix. Two consumers derive their own numbers downstream, and each
+needs the OTHER side of the split:
+
+* ``telemetry.session._fold`` accumulates ``tokens.input``, a billed/volume figure
+ that sits beside ``output``/``cache_read``/``cache_write``/``uncached`` — all
+ provider-reported. It must prefer the provider count.
+* ``PrometheusMetrics.record_request`` reconstructs the durable savings ledger as
+ ``tokens_before = input_tokens + tokens_saved``, ``tokens_after = input_tokens``.
+ That is a DELTA, so pairing a provider ``input_tokens`` with a locally-counted
+ ``tokens_saved`` straddles two rulers — local 10->6 with the provider reporting
+ 8 would record 12->8.
+
+These drive the real funnel with one deliberately mismatched pair and assert both
+semantics, rather than asserting on the dataclass alone.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.proxy.outcome import RequestOutcome
+
+# Local 10 -> 6 (saved 4); the provider says the prompt was 8. Every number below
+# is derived from exactly this one mismatch.
+_LOCAL_ORIGINAL = 10
+_LOCAL_OPTIMIZED = 6
+_LOCAL_SAVED = 4
+_PROVIDER_INPUT = 8
+
+
+def _outcome() -> RequestOutcome:
+ return RequestOutcome(
+ request_id="r1",
+ provider="openai",
+ model="gpt-4o-mini",
+ original_tokens=_LOCAL_ORIGINAL,
+ optimized_tokens=_LOCAL_OPTIMIZED,
+ provider_input_tokens=_PROVIDER_INPUT,
+ output_tokens=5,
+ tokens_saved=_LOCAL_SAVED,
+ attempted_input_tokens=_LOCAL_OPTIMIZED + _LOCAL_SAVED,
+ )
+
+
+def test_beacon_input_is_the_billed_provider_count() -> None:
+ """``tokens.input`` is a volume figure and must not silently become local."""
+ from headroom.telemetry import session as sess_mod
+
+ sess = sess_mod._Session(sid="s1", started=0.0, last_seen=0.0) # type: ignore[attr-defined]
+ sess_mod._fold(sess, _outcome(), now=0.0, source="proxy") # type: ignore[attr-defined]
+
+ assert sess.input_tokens == _PROVIDER_INPUT, (
+ "the beacon's input volume must use the provider count, not the local one"
+ )
+ # The local pair still drives the reduction ratios.
+ assert sess.original_tokens == _LOCAL_ORIGINAL
+ assert sess.tokens_saved == _LOCAL_SAVED
+
+
+def test_beacon_falls_back_to_local_when_no_provider_count() -> None:
+ """Providers that report no usage must behave exactly as before the split."""
+ from headroom.telemetry import session as sess_mod
+
+ o = RequestOutcome(
+ request_id="r2",
+ provider="anthropic",
+ model="claude-sonnet-4-6",
+ original_tokens=_LOCAL_ORIGINAL,
+ optimized_tokens=_LOCAL_OPTIMIZED,
+ output_tokens=5,
+ tokens_saved=_LOCAL_SAVED,
+ attempted_input_tokens=_LOCAL_OPTIMIZED + _LOCAL_SAVED,
+ )
+ sess = sess_mod._Session(sid="s2", started=0.0, last_seen=0.0) # type: ignore[attr-defined]
+ sess_mod._fold(sess, o, now=0.0, source="proxy") # type: ignore[attr-defined]
+
+ assert sess.input_tokens == _LOCAL_OPTIMIZED
+
+
+async def test_ledger_delta_stays_on_the_local_ruler(monkeypatch) -> None:
+ """Drive the real record_request and capture what reaches the ledger.
+
+ The ledger stores a delta, so both ends must be local. With the billed input
+ (8, provider) paired against a local tokens_saved (4), the old shape recorded
+ 12 -> 8 for a request that actually went 10 -> 6.
+ """
+ from headroom.proxy import prometheus_metrics as pm
+
+ seen: dict = {}
+
+ def fake_record_savings_event(**kw):
+ seen.update(kw)
+
+ monkeypatch.setattr(pm.savings_ledger, "record_savings_event", fake_record_savings_event)
+
+ metrics = pm.PrometheusMetrics()
+ metrics._stateless = False # the ledger write is skipped when stateless
+
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-4o-mini",
+ input_tokens=_PROVIDER_INPUT, # billed/volume: provider's count
+ local_input_tokens=_LOCAL_OPTIMIZED, # same ruler as tokens_saved
+ output_tokens=5,
+ tokens_saved=_LOCAL_SAVED,
+ latency_ms=1.0,
+ )
+
+ assert seen, "no ledger event recorded"
+ assert seen["tokens_before"] == _LOCAL_ORIGINAL, "before must be the local original"
+ assert seen["tokens_after"] == _LOCAL_OPTIMIZED, "after must be the local optimized"
+ # The mixed-ruler shape this replaces.
+ assert (seen["tokens_before"], seen["tokens_after"]) != (
+ _PROVIDER_INPUT + _LOCAL_SAVED,
+ _PROVIDER_INPUT,
+ )
+ # Volume still counted on the billed figure.
+ assert metrics.tokens_input_total == _PROVIDER_INPUT
+
+
+async def test_ledger_falls_back_to_billed_when_local_omitted(monkeypatch) -> None:
+ """Pre-split callers keep their existing (single-value) behaviour."""
+ from headroom.proxy import prometheus_metrics as pm
+
+ seen: dict = {}
+ monkeypatch.setattr(pm.savings_ledger, "record_savings_event", lambda **kw: seen.update(kw))
+ metrics = pm.PrometheusMetrics()
+ metrics._stateless = False
+
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-4o-mini",
+ input_tokens=_PROVIDER_INPUT,
+ output_tokens=5,
+ tokens_saved=_LOCAL_SAVED,
+ latency_ms=1.0,
+ )
+
+ assert seen["tokens_after"] == _PROVIDER_INPUT
+
+
+def test_record_request_defaults_local_to_billed_when_omitted() -> None:
+ """Callers that never pass local_input_tokens keep pre-split behaviour."""
+ import inspect
+
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
+
+ sig = inspect.signature(PrometheusMetrics.record_request)
+ assert sig.parameters["local_input_tokens"].default is None
diff --git a/tests/test_outcome_token_scale.py b/tests/test_outcome_token_scale.py
new file mode 100644
index 000000000..c6153bc28
--- /dev/null
+++ b/tests/test_outcome_token_scale.py
@@ -0,0 +1,97 @@
+"""``original_tokens`` and ``optimized_tokens`` must share a tokenizer scale.
+
+Every derived quantity is a delta between the two — ``tokens_saved``,
+``tokens_inflated``, ``attempted_input_tokens``, and the beacon's
+``eligible_pct`` / ``yield_pct``. Handlers used to pass the provider's
+``usage.prompt_tokens`` as ``optimized_tokens`` because it also fed billing,
+which put a provider count against a locally-estimated ``original_tokens``.
+
+A real beacon payload from a gpt-4o-mini session, where our estimator
+undercounted by 2 tokens on a 10-token request:
+
+ "tokens": {"original": 10, "attempted": 12, "input": 12, "saved": 0}
+ "rates": {"eligible_pct": 120}
+
+120% is structurally impossible — you cannot attempt to compress more than
+arrived. The same mismatch also produces a phantom ``tok_inflated``.
+"""
+
+from __future__ import annotations
+
+from headroom.proxy.outcome import RequestOutcome
+
+# The exact numbers from the reported beacon entry.
+_LOCAL_ORIGINAL = 10
+_LOCAL_OPTIMIZED = 10 # nothing compressed: a 10-token request is below every floor
+_PROVIDER_COUNT = 12 # gpt-4o-mini's own prompt_tokens — a different ruler
+
+
+def _outcome(**kw) -> RequestOutcome:
+ base: dict = {
+ "request_id": "r1",
+ "provider": "openai",
+ "model": "gpt-4o-mini",
+ "original_tokens": _LOCAL_ORIGINAL,
+ "optimized_tokens": _LOCAL_OPTIMIZED,
+ "output_tokens": 5,
+ "tokens_saved": 0,
+ "attempted_input_tokens": _LOCAL_OPTIMIZED,
+ }
+ base.update(kw)
+ return RequestOutcome(**base)
+
+
+def test_provider_count_does_not_contaminate_the_local_pair() -> None:
+ """The provider's number rides alongside instead of replacing tok_after."""
+ o = _outcome(provider_input_tokens=_PROVIDER_COUNT)
+
+ assert o.original_tokens == _LOCAL_ORIGINAL
+ assert o.optimized_tokens == _LOCAL_OPTIMIZED
+ assert o.provider_input_tokens == _PROVIDER_COUNT
+
+
+def test_no_phantom_inflation_when_the_provider_counts_higher() -> None:
+ """The regression: provider 12 vs local 10 reported 2 tokens of growth."""
+ fixed = _outcome(provider_input_tokens=_PROVIDER_COUNT)
+ assert fixed.tokens_inflated == 0
+
+ # What the old shape produced — optimized carrying the provider count.
+ contaminated = _outcome(optimized_tokens=_PROVIDER_COUNT)
+ assert contaminated.tokens_inflated == 2, (
+ "guard is inverted; this asserts the OLD behaviour to document the bug"
+ )
+
+
+def test_attempted_cannot_exceed_original_on_a_no_op_turn() -> None:
+ """`eligible_pct = attempted / original` must stay <= 100 here.
+
+ attempted is built as optimized + saved, so once optimized is local the
+ ratio is bounded by original for any turn that did not grow.
+ """
+ o = _outcome(
+ provider_input_tokens=_PROVIDER_COUNT,
+ attempted_input_tokens=_LOCAL_OPTIMIZED + 0,
+ )
+ assert o.attempted_input_tokens <= o.original_tokens
+ assert 100 * o.attempted_input_tokens / o.original_tokens == 100.0
+
+
+def test_provider_count_is_optional_and_defaults_to_zero() -> None:
+ """18 pre-existing emit sites pass nothing; billing must fall back."""
+ o = _outcome()
+ assert o.provider_input_tokens == 0
+
+
+def test_real_inflation_is_still_reported() -> None:
+ """Fixing the scale must not mute genuine post-compression growth.
+
+ Same tokenizer both sides, request forwarded larger (memory injection /
+ proactive expansion) — that is real and must survive.
+ """
+ o = _outcome(
+ original_tokens=55_161,
+ optimized_tokens=57_845,
+ provider_input_tokens=58_000,
+ tokens_saved=0,
+ )
+ assert o.tokens_inflated == 2_684