Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): show turn-hook savings in /stats extension_savings, incl. Codex WS (#4233)

Tejas Chopra1d112cd36c9a
diff --git a/headroom/observability/metrics.py b/headroom/observability/metrics.py
index 3f3592652..12c762352 100644
--- a/headroom/observability/metrics.py
+++ b/headroom/observability/metrics.py
@@ -14,6 +14,7 @@ from opentelemetry.metrics import CallbackOptions, Observation
from headroom._version import get_version
from headroom.offline import guard_egress
+from headroom.proxy.savings_attribution import event_count
logger = logging.getLogger(__name__)
@@ -558,7 +559,7 @@ class HeadroomOtelMetrics:
realized=bool(item.get("realized", True)),
estimated=bool(item.get("estimated", False)),
)
- self._savings_attribution_events.add(1, attrs)
+ self._savings_attribution_events.add(event_count(item), attrs)
saved = max(0, int(item.get("tokens", 0) or 0))
if saved:
self._savings_attributed_tokens.add(saved, attrs)
diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py
index de32f4aca..b09560aea 100644
--- a/headroom/perf/analyzer.py
+++ b/headroom/perf/analyzer.py
@@ -813,13 +813,15 @@ def format_report(report: PerfReport) -> str:
# exactly zero tokens; every token-savings channel in the proxy would record
# it as nothing. Showing `$` beside a `0 tokens` row is the honest rendering
# of that, and collapsing them into one number would be an invented saving.
+ from headroom.proxy.savings_attribution import event_count
+
by_source: dict[tuple[str, bool], dict[str, float]] = {}
for record in report.perf_records:
for item in getattr(record, "savings_breakdown", ()) or ():
source = str(item.get("source") or "other")
realized = bool(item.get("realized", True))
row = by_source.setdefault((source, realized), {"events": 0, "tokens": 0, "usd": 0.0})
- row["events"] += 1
+ row["events"] += event_count(item)
row["tokens"] += max(0, _as_number(item.get("tokens"), int))
row["usd"] += _as_number(item.get("usd"), float)
if by_source:
@@ -1356,6 +1358,8 @@ def build_perf_summary(report: PerfReport) -> dict:
}
)
+ from headroom.proxy.savings_attribution import event_count
+
by_source_groups: dict[tuple[str, bool], dict[str, int | float | str | bool]] = {}
for record in records:
for item in record.savings_breakdown:
@@ -1372,7 +1376,7 @@ def build_perf_summary(report: PerfReport) -> dict:
"usd": 0.0,
},
)
- row["events"] = int(row["events"]) + 1
+ row["events"] = int(row["events"]) + event_count(item)
raw_tokens = item.get("tokens", 0)
raw_usd = item.get("usd", 0.0)
tokens = int(raw_tokens) if isinstance(raw_tokens, (str, int, float)) else 0
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 3b1f62834..559fbc6ee 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -46,6 +46,9 @@ from headroom.proxy.provider_usage import (
)
from headroom.proxy.rate_limit_identity import rate_limit_identity
from headroom.proxy.savings import ItemMeasure, get_item_ledger, prefix_key, responses_items
+from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG
+from headroom.proxy.savings_attribution import accumulate as accumulate_savings
+from headroom.proxy.savings_attribution import from_tags as savings_from_tags
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
from headroom.proxy.tool_schema_savings_policy import (
@@ -7182,6 +7185,12 @@ class OpenAIHandlerMixin:
# handshake. Returns `{}` when no tags are present.
_extract_ws_tags = getattr(self, "_extract_tags", None)
ws_tags = _extract_ws_tags(ws_headers) if callable(_extract_ws_tags) else {}
+ # Savings attribution rows (turn hooks, tool-search deferral) from frames
+ # that were rewritten and sent, waiting for the next turn's outcome. Each
+ # frame compresses into its own dict: a timed-out compression thread
+ # keeps running and must not book rows for a frame sent uncompressed.
+ # Coalesced per source by ``accumulate_savings``, so it stays small.
+ ws_turn_savings: list[dict[str, Any]] = []
# Extract subprotocol from client — this is an application-level negotiation
# that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
@@ -8193,6 +8202,7 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_preflight_ms)
_compression_started = time.perf_counter()
+ _frame_savings: dict[str, Any] = {}
try:
(
_new_inner,
@@ -8212,6 +8222,7 @@ class OpenAIHandlerMixin:
if client == "codex"
else COMPRESSION_TIMEOUT_SECONDS,
client=client,
+ savings_tags=_frame_savings,
)
for _timing_name, _timing_ms in _ws_compression_timing.items():
_record_ws_compression_timing(_timing_name, _timing_ms)
@@ -8260,6 +8271,9 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_rewrite_ms)
tokens_saved += int(_ws_saved)
+ accumulate_savings(
+ ws_turn_savings, savings_from_tags(_frame_savings)
+ )
ws_conversation_tokens_saved = int(_ws_saved)
ws_conversation_key = savings_conversation_key(
_send_body, session_id=f"ws:{session_id}"
@@ -8586,6 +8600,7 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_preflight_ms)
_compression_started = time.perf_counter()
+ frame_savings: dict[str, Any] = {}
try:
(
new_inner,
@@ -8605,6 +8620,7 @@ class OpenAIHandlerMixin:
if client == "codex"
else COMPRESSION_TIMEOUT_SECONDS,
client=client,
+ savings_tags=frame_savings,
)
for _timing_name, _timing_ms in frame_compression_timing.items():
_record_ws_compression_timing(_timing_name, _timing_ms)
@@ -8754,6 +8770,7 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_rewrite_ms)
tokens_saved += int(frame_saved)
+ accumulate_savings(ws_turn_savings, savings_from_tags(frame_savings))
ws_conversation_tokens_saved = int(frame_saved)
ws_conversation_key = savings_conversation_key(
new_inner, session_id=f"ws:{session_id}"
@@ -9110,6 +9127,15 @@ class OpenAIHandlerMixin:
# at the WS upgrade) so dashboards can
# slice WS turns by tag — same surface
# as HTTP turns.
+ # Same gate as ``saved_delta``: a usage-less turn
+ # leaves its rows pending for a usage-carrying one.
+ turn_tags = ws_tags
+ if ws_turn_savings and input_delta > 0:
+ turn_tags = {
+ **ws_tags,
+ SAVINGS_ATTRIBUTION_TAG: list(ws_turn_savings),
+ }
+ ws_turn_savings.clear()
await self._record_request_outcome(
RequestOutcome(
# Per-emission ids keep dashboard request-log keys unique.
@@ -9140,7 +9166,7 @@ class OpenAIHandlerMixin:
)
if isinstance(body, dict)
else 0,
- tags=ws_tags,
+ tags=turn_tags,
conversation_key=ws_conversation_key,
conversation_tokens_saved=ws_conversation_tokens_saved,
client=client,
@@ -9755,6 +9781,10 @@ class OpenAIHandlerMixin:
"cache_write_tokens": str(ws_cache_write_tokens_total),
"uncached_input_tokens": str(ws_uncached_input_tokens_total),
}
+ if ws_turn_savings and residual_input_tokens > 0:
+ # Rows no per-turn record claimed ride with the residual, under
+ # the same input-usage gate as ``residual_tokens_saved``.
+ ws_session_tags[SAVINGS_ATTRIBUTION_TAG] = list(ws_turn_savings)
if (
residual_input_tokens > 0
or residual_output_tokens > 0
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index a2b355dfb..a2534803d 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
from headroom import savings_ledger
from headroom.observability import get_otel_metrics
+from headroom.proxy.savings_attribution import event_count
from headroom.proxy.savings_tracker import SavingsTracker, estimate_request_savings_usd
logger = logging.getLogger("headroom.proxy")
@@ -654,6 +655,24 @@ class PrometheusMetrics:
if saved > 0:
self.extension_savings[key] += saved
+ def extension_savings_by_source(self) -> dict[str, int]:
+ """Measured savings per source, as ``/stats`` reports ``extension_savings``.
+
+ Realized savings attribution rows (``ctx.record_savings`` from a turn
+ hook, ``record_scope_savings``) plus ``record_extension_savings`` totals.
+ A source reported through both APIs (``headroom-hdr`` does, per request)
+ shows the larger of the two totals rather than their sum: nothing ties an
+ attribution row to a ``record_extension_savings`` call, so summing would
+ double-count it. Estimated rows (``realized=False``) stay in
+ ``savings.by_source`` only.
+ """
+ out = dict(self.extension_savings)
+ for row in list(self.savings_by_source.values()):
+ source, tokens = str(row["source"]), int(row["tokens"])
+ if row["realized"] and tokens > out.get(source, 0):
+ out[source] = tokens
+ return out
+
def record_compression_failed(self, reason: str) -> None:
"""Record one fail-open compression failure, bucketed by ``reason``.
@@ -970,7 +989,7 @@ class PrometheusMetrics:
"usd": 0.0,
},
)
- row["events"] = int(row["events"]) + 1
+ row["events"] = int(row["events"]) + event_count(item)
row["tokens"] = int(row["tokens"]) + max(0, int(item.get("tokens", 0) or 0))
row["usd"] = round(
float(row["usd"]) + float(item.get("usd", 0.0) or 0.0),
diff --git a/headroom/proxy/savings_attribution.py b/headroom/proxy/savings_attribution.py
index ad64012dd..891b6fe04 100644
--- a/headroom/proxy/savings_attribution.py
+++ b/headroom/proxy/savings_attribution.py
@@ -6,12 +6,12 @@ import base64
import json
import math
import re
-from collections.abc import MutableMapping
+import warnings
+from collections.abc import Iterable, MutableMapping
from typing import Any
SAVINGS_ATTRIBUTION_TAG = "_headroom_savings_attribution"
_NAME_RE = re.compile(r"[^a-z0-9_.-]+")
-MAX_SOURCES = 32
_SCOPE_KEY = "headroom_savings_attribution"
# Per-request stage timings contributed from outside the handler, merged into
@@ -51,6 +51,22 @@ STAGE_PREFIX = "ext:"
# (16 stages x 1h is nowhere near the float ceiling).
MAX_STAGE_MS = 3_600_000.0
+# Source names are extension-supplied, so a ledger keeps at most this many
+# rows by name. A source past it folds into ``other`` -- one row per
+# (realized, estimated) -- so storage and the per-insert scan stay bounded
+# while every token, dollar and event is still booked.
+MAX_NAMED_ROWS = 256
+OVERFLOW_SOURCE = "other"
+# Core-booked sources never fold: ``tool_schema_savings_policy`` finds the
+# deferral's row by name to lower it when hooks un-defer tools.
+_UNFOLDED_SOURCES = frozenset({"tool_search"})
+
+# A row's sums saturate here instead of overflowing to ``inf`` (see
+# MAX_STAGE_MS for what a non-finite value does downstream). Both are far
+# beyond any real saving; a saturated row is flagged ``saturated``.
+MAX_ROW_USD = 1e15
+MAX_ROW_TOKENS = 10**15
+
def _source_name(value: object) -> str:
name = _NAME_RE.sub("_", str(value or "other").strip().lower()).strip("_.-")
@@ -164,9 +180,6 @@ def record_savings(
details: dict[str, Any] | None = None,
) -> None:
"""Attribute savings to a source; this never changes headline totals."""
- ledger = _ledger(tags)
- if len(ledger) >= MAX_SOURCES:
- return
# Same hazard as MAX_STAGE_MS, on the amounts rather than the durations:
# ``usd=inf`` reaches ``/stats`` and raises out of the JSON encoder, and
# ``int(inf)`` raises OverflowError right here, inside the handler, on a
@@ -193,14 +206,106 @@ def record_savings(
for key, value in list(details.items())[:12]
if isinstance(value, (str, int, float, bool)) or value is None
}
- ledger.append(item)
+ accumulate(_ledger(tags), [item])
def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]:
raw = (tags or {}).get(SAVINGS_ATTRIBUTION_TAG)
if not isinstance(raw, list):
return []
- return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)]
+ return [dict(item) for item in raw if isinstance(item, dict)]
+
+
+def event_count(row: dict[str, Any]) -> int:
+ """How many recordings a row stands for; a row without ``events`` is one."""
+ try:
+ return max(1, int(row.get("events", 1)))
+ except (TypeError, ValueError, OverflowError):
+ return 1
+
+
+def accumulate(rows: list[dict[str, Any]], items: Iterable[dict[str, Any]]) -> None:
+ """Add ``items`` to ``rows``, one row per (source, realized, estimated).
+
+ Every writer goes through here -- the per-request ledger and a WS session's
+ pending rows alike -- and no amount is ever dropped. Same-key rows sum
+ ``tokens``, ``usd`` and ``events``; ``details`` stay only while every
+ summed row agrees. Once ``rows`` holds ``MAX_NAMED_ROWS`` rows, a new
+ source folds into ``OVERFLOW_SOURCE`` (which also takes unnamed savings),
+ so the list grows by at most the overflow and ``_UNFOLDED_SOURCES`` keys.
+ """
+ for item in items:
+ key = _key(item)
+ target = _row_for(rows, key)
+ if (
+ target is None
+ and key[0] not in _UNFOLDED_SOURCES
+ and _row_count(rows) >= MAX_NAMED_ROWS
+ ):
+ if key[0] != OVERFLOW_SOURCE:
+ # Details describe one source; the overflow row mixes several.
+ item = {k: v for k, v in item.items() if k != "details"}
+ key = (OVERFLOW_SOURCE, key[1], key[2])
+ target = _row_for(rows, key)
+ if target is None:
+ target = {**item, "source": key[0], "tokens": 0, "usd": 0.0, "events": 0}
+ rows.append(target)
+ else:
+ if target.get("details") != item.get("details"):
+ target.pop("details", None)
+ target["events"] = event_count(target)
+ _add_into(target, item)
+
+
+def _add_into(target: dict[str, Any], item: dict[str, Any]) -> None:
+ """Sum ``item`` into ``target``; a total that hit a bound stays flagged."""
+ tokens, tokens_clipped = _bounded(target.get("tokens"), item.get("tokens"), 0, MAX_ROW_TOKENS)
+ usd, usd_clipped = _bounded(target.get("usd"), item.get("usd"), -MAX_ROW_USD, MAX_ROW_USD)
+ target["tokens"] = int(tokens)
+ target["usd"] = round(usd, 12)
+ target["events"] += event_count(item)
+ if tokens_clipped or usd_clipped or item.get("saturated"):
+ target["saturated"] = True
+
+
+def _bounded(current: Any, added: Any, low: float, high: float) -> tuple[float, bool]:
+ """``current + added`` held inside ``[low, high]``, and whether it was clipped."""
+ a, a_clipped = _operand(current, low, high)
+ b, b_clipped = _operand(added, low, high)
+ total = max(low, min(a + b, high))
+ return total, a_clipped or b_clipped or total != a + b
+
+
+def _operand(value: Any, low: float, high: float) -> tuple[float, bool]:
+ # Rows are plain dicts a reader may edit (deferral reconcile, plugins), so
+ # an operand can be junk; summing must not raise inside the handler. A
+ # non-finite or oversized operand reads as the bound it passed, never as
+ # zero, so a saturated total is not reset by the next update.
+ try:
+ number = float(value or 0)
+ except OverflowError: # an int beyond float range
+ number = math.inf if value > 0 else -math.inf
+ except (TypeError, ValueError):
+ return 0.0, False
+ if math.isnan(number):
+ return 0.0, False
+ bounded = max(low, min(number, high))
+ return bounded, bounded != number
+
+
+def _row_count(rows: list[Any]) -> int:
+ return sum(1 for row in rows if isinstance(row, dict))
+
+
+def _key(row: dict[str, Any]) -> tuple[str, bool, bool]:
+ # ``realized`` defaults to True, as in every consumer of these rows.
+ return (str(row.get("source")), bool(row.get("realized", True)), bool(row.get("estimated")))
+
+
+def _row_for(rows: list[dict[str, Any]], key: tuple[str, bool, bool]) -> dict[str, Any] | None:
+ # The ledger is a plain list reachable through ``tags``; skip what this
+ # module did not write rather than raise inside the handler.
+ return next((row for row in rows if isinstance(row, dict) and _key(row) == key), None)
_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG})
@@ -246,3 +351,18 @@ def decode(value: str) -> list[dict[str, Any]]:
if isinstance(decoded, list)
else []
)
+
+
+def __getattr__(name: str) -> Any:
+ # MAX_SOURCES used to cap a request's ledger at 32 rows and silently drop
+ # the rest; rows now coalesce per source, and sources past MAX_NAMED_ROWS
+ # fold into OVERFLOW_SOURCE. Kept for importers.
+ if name == "MAX_SOURCES":
+ warnings.warn(
+ "headroom.proxy.savings_attribution.MAX_SOURCES is unused and will be "
+ "removed; rows coalesce per source, past MAX_NAMED_ROWS into 'other'.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return 32
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index b910f4257..fc84bf81a 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -5276,7 +5276,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
else {},
"compressions_by_strategy": dict(m.compressions_by_strategy),
"tokens_saved_by_strategy": dict(m.tokens_saved_by_strategy),
- "extension_savings": dict(m.extension_savings),
+ "extension_savings": m.extension_savings_by_source(),
"codex_ws": {
"units_total": m.codex_ws_units_total,
"units_modified_total": m.codex_ws_units_modified_total,
diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.py
index c77469d17..5a0209f5f 100644
--- a/headroom/proxy/tool_schema_savings_policy.py
+++ b/headroom/proxy/tool_schema_savings_policy.py
@@ -84,9 +84,10 @@ def deferred_booking(tags: object, tools: object) -> DeferredBooking | None:
``None`` when nothing is booked. The schemas are copied as they were when
booked, so a later in-place edit cannot change what the credit stands for.
- The ledger entry is the latest ``tool_search`` one marked ``estimated``:
- both the core deferral and the native deferral hook book that way, and a
- hook's own measured saving never is, so it is never the one debited.
+ The ledger entry is the ``tool_search`` row marked ``estimated``: both the
+ core deferral and the native deferral hook book that way (rows coalesce, so
+ they share it), and a hook's own measured saving never is, so it is never
+ the one debited.
"""
if not isinstance(tags, dict) or "tool_search_deferred_tokens" not in tags:
return None
diff --git a/tests/test_cli_perf_format.py b/tests/test_cli_perf_format.py
index 70de765c3..029313192 100644
--- a/tests/test_cli_perf_format.py
+++ b/tests/test_cli_perf_format.py
@@ -578,3 +578,23 @@ def test_token_source_and_dollar_source_coexist(tmp_path, monkeypatch):
def test_no_section_when_nothing_attributed(tmp_path, monkeypatch):
out = _report_for([_savings_perf_line("none")], tmp_path, monkeypatch)
assert "Savings by Source" not in out
+
+
+def test_coalesced_savings_rows_count_every_event_in_text_and_summary():
+ """A PERF row summed from 3 recordings is 3 events; a row without the field is 1."""
+ report = PerfReport(
+ perf_records=[
+ PerfRecord(
+ timestamp="2026-06-05 10:00:00,000",
+ request_id="hr_1",
+ savings_breakdown=[
+ {"source": "tool_search", "tokens": 300, "events": 3},
+ {"source": "router", "tokens": 5},
+ ],
+ )
+ ]
+ )
+
+ by_source = {row["source"]: row["events"] for row in build_perf_summary(report)["by_source"]}
+ assert by_source == {"tool_search": 3, "router": 1}
+ assert "tool_search: 3 events" in analyzer.format_report(report)
diff --git a/tests/test_compression_observability.py b/tests/test_compression_observability.py
index d7c66d4dc..c96d2f7b2 100644
--- a/tests/test_compression_observability.py
+++ b/tests/test_compression_observability.py
@@ -318,6 +318,50 @@ def test_extension_savings_surface_in_stats(
}
+async def test_turn_hook_savings_surface_in_stats_extension_savings(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """``ctx.record_savings`` from a turn hook must reach ``/stats.extension_savings``.
+
+ Before, only ``record_extension_savings`` filled the field and core never
+ calls it, so ``/stats`` showed ``{}`` while a hook saved ~8K tokens a request.
+ Estimated rows stay out, and a plugin reporting through both APIs (hdr) is
+ not counted twice.
+ """
+ from fastapi.testclient import TestClient
+
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))
+ app = create_app(ProxyConfig(cache_enabled=False, rate_limit_enabled=False, log_requests=False))
+ with TestClient(app) as client:
+ metrics = app.state.proxy.metrics
+ for _ in range(2):
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-5.4",
+ input_tokens=1000,
+ output_tokens=10,
+ tokens_saved=0,
+ latency_ms=1.0,
+ savings_attribution=[
+ {"source": "tool_search", "realized": True, "tokens": 8000},
+ {"source": "tool_search", "realized": False, "estimated": True, "tokens": 500},
+ {"source": "hdr", "realized": True, "tokens": 300},
+ ],
+ )
+ metrics.record_extension_savings("hdr", 300)
+
+ stats = client.get("/stats").json()
+
+ assert stats["extension_savings"] == {"tool_search": 16000, "hdr": 600}
+ by_source = {
+ (row["source"], row["realized"]): row["tokens"] for row in stats["savings"]["by_source"]
+ }
+ assert by_source[("tool_search", False)] == 1000
+
+
def test_prometheus_metrics_accumulates_codex_ws_unit_and_frame_counters():
from headroom.proxy.prometheus_metrics import PrometheusMetrics
diff --git a/tests/test_extension_attribution.py b/tests/test_extension_attribution.py
index 77deb8f61..5a2822a47 100644
--- a/tests/test_extension_attribution.py
+++ b/tests/test_extension_attribution.py
@@ -396,3 +396,157 @@ async def test_extension_savings_reach_the_metrics_call() -> None:
attribution = h.metrics.record_request.await_args.kwargs["savings_attribution"]
assert [(row["source"], row["usd"]) for row in attribution] == [("routemegood", 0.42)]
+
+
+# --- no row cap: every recording is booked -------------------------------------
+
+
+async def test_a_request_with_40_sources_books_every_token() -> None:
+ """The ledger used to stop at 32 rows and drop the 33rd recording on."""
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
+ from headroom.proxy.savings_attribution import record_savings
+
+ tags: dict = {}
+ for i in range(40):
+ record_savings(tags, f"plugin{i}", tokens=100, usd=0.01)
+ record_savings(tags, "plugin0", tokens=100, usd=0.01)
+
+ rows = from_tags(tags)
+ metrics = PrometheusMetrics()
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-5.4",
+ input_tokens=10,
+ output_tokens=1,
+ tokens_saved=0,
+ latency_ms=1.0,
+ savings_attribution=rows,
+ )
+
+ assert len(rows) == 40
+ booked = metrics.savings_by_source.values()
+ assert sum(int(row["tokens"]) for row in booked) == 4_100
+ assert sum(int(row["events"]) for row in booked) == 41
+ assert metrics.savings_by_source["plugin0:1"]["events"] == 2
+
+
+def test_rows_coalesce_per_source_realized_and_estimated() -> None:
+ from headroom.proxy.savings_attribution import record_savings
+
+ tags: dict = {}
+ for _ in range(3):
+ record_savings(tags, "router", tokens=1, realized=True)
+ record_savings(tags, "router", tokens=1, realized=False)
+ record_savings(tags, "router", tokens=1, estimated=True)
+
+ rows = from_tags(tags)
+ assert [(r["realized"], r["estimated"], r["tokens"], r["events"]) for r in rows] == [
+ (True, False, 3, 3),
+ (False, False, 3, 3),
+ (True, True, 3, 3),
+ ]
+
+
+def test_merging_into_a_junk_row_does_not_raise_inside_the_handler() -> None:
+ from headroom.proxy.savings_attribution import record_savings
+
+ tags: dict = {SAVINGS_ATTRIBUTION_TAG: [{"source": "x", "tokens": None, "usd": "junk"}]}
+ record_savings(tags, "x", tokens=5, usd=0.5)
+
+ assert [(r["tokens"], r["usd"], r["events"]) for r in from_tags(tags)] == [(5, 0.5, 2)]
+
+
+def test_distinct_sources_past_the_bound_fold_into_other_and_conserve_totals() -> None:
+ """Source names are plugin-supplied; their count must not grow the ledger."""
+ from headroom.proxy.savings_attribution import MAX_NAMED_ROWS, record_savings
+
+ tags: dict = {}
+ for i in range(2_000):
+ record_savings(tags, f"extension_{i}", tokens=100, usd=0.01)
+ record_savings(tags, f"extension_{i}", tokens=1, estimated=True)
+ record_savings(tags, "extension_0", tokens=100, usd=0.01) # a named row still merges
+
+ rows = from_tags(tags)
+ assert len(rows) == MAX_NAMED_ROWS + 2 # plus one ``other`` row per estimated flag
+ assert sum(r["tokens"] for r in rows) == 2_000 * 101 + 100
+ assert sum(r["events"] for r in rows) == 4_001
+ assert math.isclose(sum(r["usd"] for r in rows), 2_001 * 0.01)
+ named = {(r["source"], r["estimated"]): r for r in rows}
+ assert named[("extension_0", False)]["events"] == 2
+ assert {(r["source"], r["estimated"]) for r in rows if r["source"] == "other"} == {
+ ("other", False),
+ ("other", True),
+ }
+
+
+async def test_same_source_usd_saturates_instead_of_overflowing() -> None:
+ """Two finite amounts must not sum to ``inf`` on the way to the metrics."""
+ import json
+
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
+ from headroom.proxy.savings_attribution import MAX_ROW_USD, record_savings
+
+ tags: dict = {}
+ record_savings(tags, "same", tokens=1, usd=1e308)
+ record_savings(tags, "same", tokens=1, usd=1e308)
+ record_savings(tags, "same", tokens=1, usd=1.0) # must not reset the total
+
+ rows = from_tags(tags)
+ assert [(r["tokens"], r["events"], r["usd"], r["saturated"]) for r in rows] == [
+ (3, 3, MAX_ROW_USD, True)
+ ]
+ metrics = PrometheusMetrics()
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-5.4",
+ input_tokens=10,
+ output_tokens=1,
+ tokens_saved=0,
+ latency_ms=1.0,
+ savings_attribution=rows,
+ )
+ booked = metrics.savings_by_source["same:1"]["usd"]
+ assert math.isfinite(booked)
+ json.dumps(metrics.savings_by_source, allow_nan=False) # what /stats does
+
+
[email protected]("then_usd", [1.0, -1.0])
+def test_an_infinite_total_already_in_the_ledger_stays_saturated(then_usd) -> None:
+ from headroom.proxy.savings_attribution import MAX_ROW_USD, record_savings
+
+ tags: dict = {SAVINGS_ATTRIBUTION_TAG: [{"source": "x", "usd": math.inf}]}
+ record_savings(tags, "x", usd=then_usd)
+
+ [row] = from_tags(tags)
+ assert row["saturated"] is True
+ assert row["usd"] >= MAX_ROW_USD - 1.0
+
+
+def test_tokens_stay_within_zero_and_the_bound() -> None:
+ from headroom.proxy.savings_attribution import MAX_ROW_TOKENS, accumulate
+
+ rows: list = [{"source": "x", "tokens": -(10**400)}, {"source": "y", "tokens": 10**400}]
+ accumulate(rows, [{"source": "x", "tokens": 5}, {"source": "y", "tokens": 5}])
+
+ assert [(r["tokens"], r["saturated"]) for r in rows] == [(5, True), (MAX_ROW_TOKENS, True)]
+
+
+def test_tool_search_keeps_its_own_row_past_the_bound() -> None:
+ """Deferral reconcile finds the ``tool_search`` row by name to lower it."""
+ from headroom.proxy.savings_attribution import MAX_NAMED_ROWS, record_savings
+
+ tags: dict = {}
+ for i in range(MAX_NAMED_ROWS):
+ record_savings(tags, f"extension_{i}", tokens=1)
+ record_savings(tags, "tool_search", tokens=900, realized=False, estimated=True)
+
+ assert [r["tokens"] for r in from_tags(tags) if r["source"] == "tool_search"] == [900]
+
+
+def test_max_sources_is_deprecated_not_removed() -> None:
+ from headroom.proxy import savings_attribution
+
+ with pytest.warns(DeprecationWarning, match="MAX_SOURCES"):
+ assert savings_attribution.MAX_SOURCES == 32
diff --git a/tests/test_observability_metrics.py b/tests/test_observability_metrics.py
index 6aea727e8..21d3c1265 100644
--- a/tests/test_observability_metrics.py
+++ b/tests/test_observability_metrics.py
@@ -262,6 +262,23 @@ def test_request_attribute_provider_enriches_core_and_savings_metrics() -> None:
unregister_otel_metric_attribute_provider(identity_attributes)
+def test_coalesced_savings_rows_count_every_event() -> None:
+ """A row summed from 33 recordings is 33 events, not 1; a bare row is 1."""
+ reader = InMemoryMetricReader()
+ otel_metrics = HeadroomOtelMetrics(meter_provider=MeterProvider(metric_readers=[reader]))
+
+ otel_metrics.record_savings_attribution(
+ [
+ {"source": "tool_search", "tokens": 3_300, "events": 33},
+ {"source": "router", "tokens": 5},
+ ]
+ )
+
+ events = _collect_metrics(reader)["headroom.savings.attribution.events"]
+ assert _find_point(events, source="tool_search").value == 33
+ assert _find_point(events, source="router").value == 1
+
+
def test_failing_request_attribute_provider_is_fail_open() -> None:
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader])
diff --git a/tests/test_openai_codex_ws_lifecycle.py b/tests/test_openai_codex_ws_lifecycle.py
index adfdd1463..75c1cd40c 100644
--- a/tests/test_openai_codex_ws_lifecycle.py
+++ b/tests/test_openai_codex_ws_lifecycle.py
@@ -11,6 +11,7 @@ import asyncio
import contextlib
import json
import logging
+import math
import sys
from pathlib import Path
from types import SimpleNamespace
@@ -2372,3 +2373,197 @@ async def test_ws_session_metrics_track_model_per_response_create():
"model-a",
"model-b",
]
+
+
+ ("usage", "expected"),
+ [({"input_tokens": 10, "output_tokens": 1}, [("ws_fold", True)]), (None, [])],
+ ids=["billed", "usage_less"],
+)
+async def test_ws_turn_hook_savings_reach_the_turn_outcome(usage, expected):
+ """A turn hook's savings on a Codex WS turn must reach that turn's outcome.
+
+ The WS path compressed without ``savings_tags``, so ``ctx.record_savings``
+ wrote into a throwaway dict and ``/stats`` never saw the attribution. A turn
+ with no input usage books nothing, the same gate as its headline savings.
+ """
+ from headroom.proxy.savings_attribution import from_tags
+ from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
+
+ class _DropOldInput:
+ name = "drop_old_input"
+ savings_source = "ws_fold"
+ stream_safe = True
+
+ def on_request(self, ctx: SimpleNamespace) -> None:
+ del ctx.messages[:-1]
+
+ first = json.dumps(
+ {
+ "type": "response.create",
+ "response": {
+ "model": "gpt-5.4",
+ "input": [
+ {"type": "message", "role": "user", "content": "old " * 50},
+ {"type": "message", "role": "user", "content": "new"},
+ ],
+ },
+ }
+ )
+ upstream = _FakeUpstream(
+ [
+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
+ json.dumps(
+ {
+ "type": "response.completed",
+ "response": {"id": "r_1", **({"usage": usage} if usage else {})},
+ }
+ ),
+ ]
+ )
+ client_ws = _FakeWebSocket(frames=[first])
+ handler = _DummyOpenAIHandler()
+ handler.config.optimize = True
+ outcomes = []
+
+ async def _record_request_outcome(outcome):
+ outcomes.append(outcome)
+
+ handler._record_request_outcome = _record_request_outcome
+
+ clear_turn_hooks()
+ register_turn_hook(_DropOldInput())
+ try:
+ with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):
+ await handler.handle_openai_responses_ws(client_ws)
+ finally:
+ clear_turn_hooks()
+
+ sent = json.loads(upstream.sent[0])["response"]["input"]
+ assert [item["content"] for item in sent] == ["new"] # the hook really ran
+ rows = [row for outcome in outcomes for row in from_tags(outcome.tags)]
+ assert [(row["source"], row["realized"]) for row in rows] == expected
+ assert all(row["tokens"] > 0 for row in rows)
+
+
+class _UpstreamAfterSends(_FakeUpstream):
+ """Streams its events only once ``sends`` frames reached it, so every frame's
+ savings are pending before the one usage-carrying outcome claims them."""
+
+ def __init__(self, events: list[str], *, sends: int) -> None:
+ super().__init__(events)
+ self._sends = sends
+
+ async def _iter(self):
+ while len(self.sent) < self._sends:
+ await asyncio.sleep(0)
+ for ev in self._events:
+ yield ev
+
+
+ ("sources_per_frame", "frames_n", "expected_rows", "overflowed"),
+ [
+ (lambda i: "tool_search", 33, 2, False),
+ (lambda i: f"src{i}", 33, 34, False),
+ # ws_fold + 255 named sources fill MAX_NAMED_ROWS; the other 45 share one row.
+ (lambda i: f"src{i}", 300, 257, True),
+ ],
+ ids=["repeated_source", "distinct_sources", "distinct_sources_past_the_bound"],
+)
+async def test_ws_pending_savings_beyond_32_rows_are_all_booked(
+ sources_per_frame, frames_n, expected_rows, overflowed
+):
+ """Pending frames hand every token and every event to the billed turn.
+
+ Rows coalesce to one per (source, realized, estimated) key; distinct
+ sources past ``MAX_NAMED_ROWS`` fold into ``other`` rather than growing
+ the session's pending list without bound.
+ """
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
+ from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG, from_tags
+ from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
+
+ seen = {"frame": 0}
+
+ class _DeferTools:
+ name = "defer_tools"
+ savings_source = "ws_fold"
+ stream_safe = True
+
+ def on_request(self, ctx: SimpleNamespace) -> None:
+ del ctx.messages[:-1]
+ ctx.record_savings(sources_per_frame(seen["frame"]), tokens=100, usd=0.01)
+ seen["frame"] += 1
+
+ def _frame(i: int) -> str:
+ return json.dumps(
+ {
+ "type": "response.create",
+ "response": {
+ "model": "gpt-5.4",
+ "input": [
+ {"type": "message", "role": "user", "content": f"old {i} " * 50},
+ {"type": "message", "role": "user", "content": f"new {i}"},
+ ],
+ },
+ }
+ )
+
+ upstream = _UpstreamAfterSends(
+ [
+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
+ json.dumps(
+ {
+ "type": "response.completed",
+ "response": {
+ "id": "r_1",
+ "usage": {"input_tokens": 10, "output_tokens": 1},
+ },
+ }
+ ),
+ ],
+ sends=frames_n,
+ )
+ client_ws = _FakeWebSocket(frames=[_frame(i) for i in range(frames_n)])
+ handler = _DummyOpenAIHandler()
+ handler.config.optimize = True
+ outcomes = []
+
+ async def _record_request_outcome(outcome):
+ outcomes.append(outcome)
+
+ handler._record_request_outcome = _record_request_outcome
+
+ clear_turn_hooks()
+ register_turn_hook(_DeferTools())
+ try:
+ with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):
+ await handler.handle_openai_responses_ws(client_ws)
+ finally:
+ clear_turn_hooks()
+
+ assert len(upstream.sent) == frames_n
+ assert seen["frame"] == frames_n
+ metrics = PrometheusMetrics()
+ stored = []
+ for outcome in outcomes:
+ rows = from_tags(outcome.tags)
+ stored.append(len(outcome.tags.get(SAVINGS_ATTRIBUTION_TAG, [])))
+ await metrics.record_request(
+ provider="openai",
+ model="gpt-5.4",
+ input_tokens=10,
+ output_tokens=1,
+ tokens_saved=0,
+ latency_ms=1.0,
+ savings_attribution=rows,
+ )
+ hook_rows = [row for row in metrics.savings_by_source.values() if row["source"] != "ws_fold"]
+ assert sum(int(row["tokens"]) for row in hook_rows) == 100 * frames_n
+ assert sum(int(row["events"]) for row in hook_rows) == frames_n
+ assert math.isclose(sum(float(row["usd"]) for row in hook_rows), 0.01 * frames_n)
+ assert ("other" in {row["source"] for row in hook_rows}) is overflowed
+ assert stored == [expected_rows]
diff --git a/tests/test_proxy_openai.py b/tests/test_proxy_openai.py
index f7244b1ac..f677ca6d3 100644
--- a/tests/test_proxy_openai.py
+++ b/tests/test_proxy_openai.py
@@ -449,5 +449,7 @@ def test_websocket_route_carries_the_client_decision(
assert seen, "the Responses compressor was never reached on the WS path"
assert {
- k: v for k, v in seen[0].items() if k not in {"timing", "deadline_started_at"}
+ k: v
+ for k, v in seen[0].items()
+ if k not in {"timing", "deadline_started_at", "savings_tags"}
} == expected
diff --git a/tests/test_turn_hooks.py b/tests/test_turn_hooks.py
index 1bf2da3bc..35dd4a517 100644
--- a/tests/test_turn_hooks.py
+++ b/tests/test_turn_hooks.py
@@ -133,6 +133,7 @@ def test_request_runner_attributes_savings_with_handler_counters():
"tokens": 15,
"usd": 0.0,
"details": {"message_tokens_saved": 14, "tool_tokens_saved": 1},
+ "events": 1,
}
]