Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(savings): add the per-request counterfactual savings record (#4236)
Tejas Chopra2912b7b620d7
diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.pyindex 2763120fb..5f3bc65cf 100644--- a/headroom/pricing/counterfactual.py+++ b/headroom/pricing/counterfactual.py@@ -374,6 +374,26 @@ def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context:return rate if math.isfinite(rate) and rate >= 0 else 0.0+def _catalog_row(model: str) -> tuple[Any, str, dict[str, Any]] | None:+ """``(litellm, resolved model, catalog row)``, or ``None``. Never raises."""+ litellm = _litellm()+ if litellm is None:+ return None+ try:+ from headroom.pricing.litellm_pricing import resolve_litellm_model++ resolved_model = resolve_litellm_model(model)+ info = litellm.model_cost.get(resolved_model, {}) or {}+ except Exception as exc:+ logger.debug(+ "counterfactual: LiteLLM catalog lookup failed for %s: %s",+ safe_id(model),+ describe_exception(exc),+ )+ return None+ return litellm, resolved_model, info++@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)def resolve_rates(model: str,@@ -413,22 +433,10 @@ def resolve_rates(catalog between cases must call ``resolve_rates.cache_clear()`` in between,or the previous case's rates leak into the next."""- litellm = _litellm()- if litellm is None:- return None-- try:- from headroom.pricing.litellm_pricing import resolve_litellm_model-- resolved_model = resolve_litellm_model(model)- info = litellm.model_cost.get(resolved_model, {}) or {}- except Exception as exc:- logger.debug(- "counterfactual: LiteLLM catalog lookup failed for %s: %s",- safe_id(model),- describe_exception(exc),- )+ row = _catalog_row(model)+ if row is None:return None+ litellm, resolved_model, info = rowbase = info.get("input_cost_per_token")# `is None` distinguishes "unknown model" from "genuinely free". `not base`@@ -530,6 +538,19 @@ def resolve_rates()+def catalog_min_cacheable_tokens(model: str) -> int | None:+ """The catalog's smallest prompt the provider will cache for ``model``.++ LiteLLM publishes it as ``prompt_cache_min_tokens`` (Anthropic: 1024 for+ Sonnet, 4096 for Haiku 4.5). ``None`` when litellm or the row lacks it.+ """+ row = _catalog_row(model)+ value = row[2].get("prompt_cache_min_tokens") if row is not None else None+ if isinstance(value, bool) or not isinstance(value, int | float):+ return None+ return int(value) if math.isfinite(value) and value > 0 else None++@dataclass(frozen=True)class PricedSavings:"""What a set of counterfactual tokens was worth, and how sure we are.@@ -641,6 +662,7 @@ __all__ = ["PricedSavings","Region","TokenSplit",+ "catalog_min_cacheable_tokens","price_savings","resolve_rates","split_tokens",diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 4b5d299bc..3b1f62834 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -45,6 +45,7 @@ from headroom.proxy.provider_usage import (usage_int,)from headroom.proxy.rate_limit_identity import rate_limit_identity+from headroom.proxy.savings import ItemMeasure, get_item_ledger, prefix_key, responses_itemsfrom headroom.proxy.semantic_cache_key_policy import compute_request_cache_partitionfrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_logfrom headroom.proxy.tool_schema_savings_policy import (@@ -6391,6 +6392,36 @@ class OpenAIHandlerMixin:"upstream request for server-side retrieval handling")+ # Per-item original/sent counts against this conversation's previous+ # request, in Headroom's tokenizer (headroom.proxy.savings). ORIGINAL+ # is the client's bytes, before memory, compression or shaping; SENT is+ # the body forwarded below. All of it runs off the event loop: it+ # decodes, serializes and tokenizes the whole transcript. Skipped for+ # incremental bodies (previous_response_id / conversation): the+ # provider bills context they do not carry. It covers the first+ # upstream call only; continuations and re-drives are the wiring's to+ # mark as estimated. The outcome funnel settles it (ItemLedger.settle).+ savings_measure = None+ if not (body.get("previous_response_id") or body.get("conversation")):++ def _measure_items() -> ItemMeasure:+ original = responses_items(json.loads(original_body_bytes))+ return get_item_ledger().measure(+ prefix_key(_pre_compression_conversation_key, model, original),+ original,+ responses_items(body),+ tokenizer.count_text,+ )++ try:+ savings_measure = await asyncio.to_thread(_measure_items)+ except Exception as exc: # noqa: BLE001 - telemetry must never break a request+ logger.warning(+ "[%s] savings item measurement skipped: %s",+ request_id,+ describe_exception(exc),+ )+try:if stream and not buffered_stream_ccr:# Streaming for Responses API uses semantic events@@ -6415,6 +6446,7 @@ class OpenAIHandlerMixin:waste_signals=waste_signals_dict,conversation_key=responses_conversation_key,conversation_tokens_saved=tokens_saved,+ savings_measure=savings_measure,)else:@@ -6883,6 +6915,7 @@ class OpenAIHandlerMixin:tags=_resp_log_tags,conversation_key=responses_conversation_key,conversation_tokens_saved=tokens_saved,+ savings_measure=savings_measure,turn_id=compute_turn_id(model, body.get("instructions"), messages),request_messages=messagesif getattr(self.config, "log_full_messages", False)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex c8de2c75f..f7dde7aca 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -32,6 +32,8 @@ from headroom.proxy.token_counting import gemini_output_tokensif TYPE_CHECKING:from fastapi.responses import Response, StreamingResponse+ from headroom.proxy.savings import ItemMeasure+import httpx@@ -815,6 +817,7 @@ class StreamingMixin:# See ``conversation_savings``.conversation_key: str | None = None,conversation_tokens_saved: int | None = None,+ savings_measure: ItemMeasure | None = None,) -> None:from headroom.proxy.outcome import RequestOutcome@@ -1040,6 +1043,7 @@ class StreamingMixin:waste_signals=waste_signals,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ savings_measure=savings_measure,)await self._record_request_outcome(outcome)@@ -1071,6 +1075,7 @@ class StreamingMixin:session_key: str | None = None,conversation_key: str | None = None,conversation_tokens_saved: int | None = None,+ savings_measure: ItemMeasure | None = None,server_memory_tool_names: frozenset[str] | None = None,client_beta: str | None = None,) -> Response | StreamingResponse:@@ -1118,6 +1123,7 @@ class StreamingMixin:session_key=session_key,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ savings_measure=savings_measure,server_memory_tool_names=server_memory_tool_names,client_beta=client_beta,)@@ -1152,6 +1158,7 @@ class StreamingMixin:session_key: str,conversation_key: str | None = None,conversation_tokens_saved: int | None = None,+ savings_measure: ItemMeasure | None = None,server_memory_tool_names: frozenset[str] | None = None,client_beta: str | None = None,) -> Response | StreamingResponse:@@ -1586,6 +1593,7 @@ class StreamingMixin:status_code=upstream_response.status_code,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ savings_measure=savings_measure,)self._cleanup_mid_turn_stream(session_key)return Response(@@ -1981,6 +1989,7 @@ class StreamingMixin:),conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ savings_measure=savings_measure,)if supports_mid_turn_coalescing(client) and pending_messages:pending_event = json.dumps(diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 3503d710f..837544683 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -33,6 +33,7 @@ from typing import Anyfrom headroom.log_safety import describe_exceptionfrom headroom.proxy.conversation_savings import get_conversation_savings+from headroom.proxy.savings import ItemMeasure, get_item_ledgerfrom headroom.proxy.tool_schema_savings_policy import (headline_tokens_saved,tool_schema_saved_from_tags,@@ -204,6 +205,11 @@ class RequestOutcome:# path, where the outcome carries a per-turn delta.conversation_key: str | None = Noneconversation_tokens_saved: int | None = None+ # savings_measure: the request's per-item original/sent counts against the+ # conversation's previous request (``headroom.proxy.savings``), from+ # which the per-request savings record is priced. None on paths that+ # do not measure items yet.+ savings_measure: ItemMeasure | None = Nonetransforms_applied: tuple[str, ...] = ()waste_signals: dict[str, int] | None = Nonenum_messages: int = 0@@ -339,6 +345,7 @@ class RequestOutcome:conversation_key: str | None = None,conversation_tokens_saved: int | None = None,provider_input_tokens: int = 0,+ savings_measure: ItemMeasure | None = None,) -> RequestOutcome:"""Construct an outcome from the locals available at streamingfinalize. Three streaming finalizers@@ -414,6 +421,7 @@ class RequestOutcome:tokens_saved=tokens_saved,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ savings_measure=savings_measure,attempted_input_tokens=optimized_tokens + tokens_saved,status_code=status_code,cache_read_tokens=cache_read_tokens,@@ -539,6 +547,10 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:# beacon must never add latency to, or take down, the request path.record_outcome(outcome)+ if outcome.savings_measure is not None:+ # Only a request the provider served becomes the next one's baseline.+ get_item_ledger().settle(outcome.savings_measure, succeeded=outcome.status_code < 400)+# A rejected turn must not feed the savings/cost/log success stats; that# would let a request the provider never billed inflate the save-rate.# Record it under the counter that names what happened, and stop.diff --git a/headroom/proxy/savings.py b/headroom/proxy/savings.pynew file mode 100644index 000000000..69a87bea2--- /dev/null+++ b/headroom/proxy/savings.py@@ -0,0 +1,707 @@+"""One per-request savings record: counterfactual cost minus actual cost.++Every savings surface (/stats, the durable ledger, persistent savings, the+dashboard, the PERF line) is meant to derive from the :class:`SavingsRecord`+this module computes, so no surface prices savings its own way.++Two rulers meet here and are only ever combined through RATIOS:++* Headroom's tokenizer gives per-item counts of the request as the client sent+ it (ORIGINAL) and as Headroom forwarded it (SENT). Both are counted with the+ same tokenizer, so a tokenizer bias cancels in their difference.+* The provider's usage block gives exact billed totals (uncached, cache read,+ cache write) but no per-item detail.++``k = billed / sent`` converts Headroom units into provider units, calibrated+on this request's own content. The counterfactual cache behaviour mirrors the+actual request's: removed tokens in items the previous request already carried+would have been cache reads to the extent the actual request hit its cache+(``warm``); removed tokens in new items would have been misses, written to+cache in the proportion the actual request wrote (``w``). Old items whose sent+copy Headroom changed since the previous request (``bust``) are misses that+Headroom caused, charged back against the saving.++Output tokens are identical in the counterfactual by construction; output+shaping is measured separately against its holdout and stays out of this+number.++``headroom.pricing`` is imported inside functions, as elsewhere on the proxy+path: its package ``__init__`` pulls in litellm (~4s), which must not land on+proxy startup.+"""++from __future__ import annotations++import hashlib+import json+import logging+import threading+from collections import OrderedDict+from collections.abc import Callable, Sequence+from dataclasses import dataclass, field, replace+from typing import TYPE_CHECKING, Any++from headroom.log_safety import describe_exception, safe_id++if TYPE_CHECKING:+ from headroom.pricing.counterfactual import CacheMix, CacheRates++logger = logging.getLogger(__name__)++#: OpenAI caches prompts of at least 1024 tokens, in 128-token blocks.+OPENAI_MIN_CACHEABLE_TOKENS = 1024+OPENAI_CACHE_BLOCK_TOKENS = 128+#: Used when the catalog has no ``prompt_cache_min_tokens`` for a model.+DEFAULT_MIN_CACHEABLE_TOKENS = 1024+++@dataclass(frozen=True, slots=True)+class ItemCounts:+ """One request item (tools block, system, or one input/message item).++ Hashes identify the item's content; counts are in Headroom tokens. Lists of+ items are aligned by position between ORIGINAL and SENT, so an item+ Headroom dropped entirely keeps its slot with ``sent_hash=""`` and+ ``sent_tokens=0``. Tool-schema deferral is removal inside the tools item.+ """++ original_hash: str+ original_tokens: int+ sent_hash: str+ sent_tokens: int+++@dataclass(frozen=True)+class ItemMeasure:+ """The Headroom-unit quantities the savings formula needs."""++ #: ``S``: tokens of the whole sent body.+ sent_tokens: int+ #: ``R_old``: removed tokens (signed: an item Headroom grew counts against+ #: it) in items within the longest prefix of the ORIGINAL body identical to+ #: the previous ORIGINAL body.+ removed_old: int+ #: ``R_new``: removed tokens in every other item, signed likewise.+ removed_new: int+ #: ``P_S``: tokens of the longest prefix of the SENT body identical to the+ #: previous SENT body.+ sent_shared_prefix: int+ #: Sent tokens of old items that fell outside the sent shared prefix: cache+ #: misses caused only by Headroom changing what it had sent before.+ bust: int+ #: False when the old/new split is approximate (the totals stay exact):+ #: Headroom changed the item count and the original/sent pairing had to be+ #: approximated, or the baseline was lost (``baseline_lost``).+ aligned: bool = True+ #: True when the ledger lost what this conversation last sent (its+ #: baseline or an in-flight request was evicted under the budget, or its+ #: baseline may be truncated), so ``removed_new`` may hold old items. Such+ #: a measure is also ``aligned=False``. Its record prices the new removal+ #: at the cache-read rate, the lowest any input token costs, so the lost+ #: knowledge can only understate the saving. A conversation's first+ #: request is not lost: nothing was sent before, so everything is new.+ baseline_lost: bool = False+ #: Tokens Headroom added to new items (the negative part of+ #: ``removed_new``), so the read-rate floor covers only what it removed.+ grown_new: int = 0+ #: Which measured request this is, for :meth:`ItemLedger.settle`.+ ticket: int = field(default=0, compare=False)++ @property+ def removed(self) -> int:+ return self.removed_old + self.removed_new+++@dataclass(frozen=True)+class SavingsRecord:+ """What one request saved, in dollars and in both token rulers."""++ saved_usd: float+ actual_input_cost_usd: float+ counterfactual_input_cost_usd: float+ #: Removed tokens in Headroom units (``R``).+ removed_tokens: int+ #: The same removal in provider units (``k * R``).+ saved_tokens: float+ k: float+ warm: float+ write_share: float+ #: True when usage was missing (aborted stream) or summed over server-side+ #: iterations, so the cache split could not be read from this request.+ estimated: bool = False+ #: False when the catalog has no price for the model.+ priced: bool = True+++def _common_prefix_len(a: Sequence[str], b: Sequence[str]) -> int:+ n = 0+ for x, y in zip(a, b):+ if x != y:+ break+ n += 1+ return n+++def measure_items(+ items: Sequence[ItemCounts],+ previous: Sequence[ItemCounts] | None,+) -> ItemMeasure:+ """Split one request's removal into old/new items against the previous request.++ ``previous`` is the same conversation's previous request, or ``None`` on its+ first request, where everything is new and nothing is shared.+ """+ n_old = _common_prefix_len(+ [i.original_hash for i in items], [i.original_hash for i in previous or ()]+ )+ n_sent_shared = _common_prefix_len(+ [i.sent_hash for i in items], [i.sent_hash for i in previous or ()]+ )+ removed = [i.original_tokens - i.sent_tokens for i in items]+ sent = [max(0, i.sent_tokens) for i in items]+ return ItemMeasure(+ sent_tokens=sum(sent),+ removed_old=sum(removed[:n_old]),+ removed_new=sum(removed[n_old:]),+ sent_shared_prefix=sum(sent[:n_sent_shared]),+ bust=sum(sent[n_sent_shared:n_old]),+ grown_new=-sum(r for r in removed[n_old:] if r < 0),+ )+++def _dump(value: Any) -> str:+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)+++#: One request item as (identity text, billable text): the identity is hashed+#: to tell items apart, the billable text is what gets tokenized.+Item = tuple[str, str]+++def _item(value: Any) -> Item:+ identity = _dump(value)+ if (+ isinstance(value, dict)+ and value.get("type") == "reasoning"+ and "encrypted_content" in value+ ):+ # The provider bills the reasoning it decrypts, not the ciphertext,+ # which would otherwise be most of a long Codex transcript's count.+ billable = _dump({k: v for k, v in value.items() if k != "encrypted_content"})+ return identity, billable+ return identity, identity+++def responses_items(body: dict[str, Any]) -> list[Item]:+ """A ``/v1/responses`` body as items in prompt order.++ Tools, then instructions, then each ``input`` item: the order the provider+ caches the prompt in, which is what the shared-prefix comparison needs.+ Codex's ``additional_tools`` input items stay where the client put them.+ """+ items = [_item(body.get("tools") or []), _item(body.get("instructions") or "")]+ # OpenAI-compatible upstreams accept ``messages`` in place of ``input``.+ raw = body.get("input") if body.get("input") is not None else body.get("messages")+ if isinstance(raw, list):+ items.extend(_item(item) for item in raw)+ elif raw:+ items.append(_item(raw))+ return items+++def _digest(text: str) -> str:+ return hashlib.blake2b(text.encode("utf-8", errors="replace"), digest_size=8).hexdigest()+++#: Items kept across all conversations' baselines and requests awaiting their+#: outcome, combined. A long Codex session carries 1000+ items; bounding items,+#: not conversations, bounds memory: up to twice this while one conversation's+#: baseline and its in-flight request exceed it (see :class:`ItemLedger`).+MAX_STORED_ITEMS = 100_000++#: Conversations remembered as having lost their baseline, a memory bound+#: (one short key each). Past it the ledger can no longer tell a first request+#: from a lost one and treats every request without a baseline as lost: priced+#: at the cache-read floor, which can understate a saving but never overstate it.+MAX_LOST_KEYS = 100_000+++def _align(+ original: list[tuple[str, int]],+ sent: list[tuple[str, int]],+ previous: Sequence[ItemCounts] | None,+) -> tuple[list[ItemCounts], bool]:+ """Pair original and sent items, and say whether the pairing is exact.++ Same item count: by position (compression rewrites items in place). A+ different count (a hook folded or dropped items, or Headroom inserted+ some): identical items at either end pair up; the unmatched originals in+ between are removed, and the unmatched sent items in between are folded+ into the first of their slots. The totals stay exact either way; only+ which items count as old is approximate, hence ``aligned=False``. An item+ compressed in place still pairs at the head when the previous request+ paired the same original with the same sent copy at that position.++ Equal counts pair by position even if a fold and an insert cancel out.+ """+ if len(original) == len(sent):+ return [ItemCounts(oh, ot, sh, st) for (oh, ot), (sh, st) in zip(original, sent)], True+ limit = min(len(original), len(sent))+ prev = previous or ()++ def same_as_before(i: int) -> bool:+ return (+ i < len(prev)+ and original[i][0] == prev[i].original_hash+ and sent[i][0] == prev[i].sent_hash+ )++ head = 0+ while head < limit and (original[head][0] == sent[head][0] or same_as_before(head)):+ head += 1+ tail = 0+ while (+ tail < limit - head+ and original[len(original) - 1 - tail][0] == sent[len(sent) - 1 - tail][0]+ ):+ tail += 1+ mid_original = original[head : len(original) - tail]+ mid_sent = sent[head : len(sent) - tail]+ if len(mid_sent) == 1:+ folded_hash = mid_sent[0][0]+ else:+ folded_hash = _digest("\x00".join(h for h, _ in mid_sent)) if mid_sent else ""+ folded_tokens = sum(t for _, t in mid_sent)+ middle = [ItemCounts(oh, ot, "", 0) for oh, ot in mid_original] or [ItemCounts("", 0, "", 0)]+ middle[0] = ItemCounts(+ middle[0].original_hash, middle[0].original_tokens, folded_hash, folded_tokens+ )+ pairs = [ItemCounts(oh, ot, sh, st) for (oh, ot), (sh, st) in zip(original[:head], sent[:head])]+ pairs += middle+ pairs += [ItemCounts(h, t, h, t) for h, t in original[len(original) - tail :]]+ return pairs, False+++class ItemLedger:+ """The previous request of each conversation, for the old/new item split.++ One instance per process. A request is measured against the conversation's+ last SUCCESSFUL request and becomes the baseline only once its own outcome+ succeeds (:meth:`settle`): a failed call never reached the provider's cache.+ Each measure carries a ticket, issued in measure order, so a slow earlier+ turn settling late cannot replace a newer turn's baseline. Items re-sent+ from the baseline reuse its token counts, so a long transcript is+ tokenized once rather than every turn.++ Baselines and requests awaiting their outcome share one budget,+ :data:`MAX_STORED_ITEMS`. Past it, whichever is least recent goes first,+ by ticket: a pending request by its own ticket (its slot is reserved when+ the ticket is issued, so a slow measure keeps its place), a baseline by+ the last ticket that used it. Settling counts as a use. A request being+ admitted never evicts its own conversation's baseline: that baseline is+ replaced only when a newer request for the conversation succeeds. When+ only that baseline and one pending request are left, both are kept, even+ past the budget, so a conversation larger than half the budget still+ advances; memory then peaks at twice :data:`MAX_STORED_ITEMS` items.++ A request is kept only up to its first :data:`MAX_STORED_ITEMS` items: the+ comparison is a prefix, so the next request still gets the shared-prefix+ credit for them.++ Whatever is evicted, its conversation is remembered as lost, so its next+ request is marked ``baseline_lost`` (and ``aligned=False``): its record+ prices the new removal at the cache-read floor instead of as new, at the+ miss rate. So is a request measured against a baseline of exactly+ :data:`MAX_STORED_ITEMS` items, which may be truncated. A conversation's+ first request is new, not lost. The memory of losses is bounded by+ :data:`MAX_LOST_KEYS`; past it, every request without a baseline is+ treated as lost, and a forgotten loss's stale baseline is dropped with it.+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ # key -> (settled ticket, items, last ticket that used it), oldest use first.+ self._previous: OrderedDict[str, tuple[int, list[ItemCounts], int]] = OrderedDict()+ # ticket -> (key, items), in ticket order.+ self._pending: OrderedDict[int, tuple[str, list[ItemCounts]]] = OrderedDict()+ # key -> newest ticket whose items were evicted, oldest loss first.+ self._lost: OrderedDict[str, int] = OrderedDict()+ # True once a loss was forgotten: a missing baseline may then be a lost one.+ self._lost_overflowed = False+ self._stored = 0+ self._pending_items = 0+ self._seq = 0++ def measure(+ self,+ key: str,+ original: Sequence[Item],+ sent: Sequence[Item],+ count: Callable[[str], int],+ ) -> ItemMeasure:+ """Measure one request against ``key``'s baseline; settle it later."""+ with self._lock:+ self._seq += 1+ seq = self._seq+ entry = self._previous.get(key)+ if entry is not None:+ self._previous[key] = (entry[0], entry[1], seq)+ self._previous.move_to_end(key)+ lost = key in self._lost or (entry is None and self._lost_overflowed)+ self._pending[seq] = (key, [])+ previous = entry[1] if entry else None+ known: dict[str, int] = {}+ for item in previous or ():+ known[item.original_hash] = item.original_tokens+ known[item.sent_hash] = item.sent_tokens++ def counted(items: Sequence[Item]) -> list[tuple[str, int]]:+ out = []+ for identity, billable in items:+ digest = _digest(identity)+ tokens = known.get(digest)+ if tokens is None:+ tokens = known[digest] = int(count(billable))+ out.append((digest, tokens))+ return out++ try:+ items, aligned = _align(counted(original), counted(sent), previous)+ except BaseException:+ # The caller logs and never settles: release the reserved slot.+ with self._lock:+ self._pending.pop(seq, None)+ raise+ if previous is not None and len(previous) == MAX_STORED_ITEMS:+ # Possibly a truncated baseline: items past it may be old.+ lost = True+ with self._lock:+ if seq in self._pending: # not already evicted while counting+ kept = items[:MAX_STORED_ITEMS]+ self._pending[seq] = (key, kept)+ self._pending_items += len(kept)+ self._evict(key)+ return replace(+ measure_items(items, previous),+ aligned=aligned and not lost,+ baseline_lost=lost,+ ticket=seq,+ )++ def settle(self, measure: ItemMeasure, *, succeeded: bool) -> None:+ """Make a measured request its conversation's baseline if it succeeded."""+ seq = measure.ticket+ with self._lock:+ pending = self._pending.pop(seq, None)+ if pending is None:+ return+ key, items = pending+ self._pending_items -= len(items)+ if not succeeded:+ return+ current = self._previous.get(key)+ if current is not None and current[0] > seq:+ return+ # Moving items from pending to stored never grows the total.+ self._stored -= len(current[1]) if current else 0+ self._previous[key] = (seq, items, self._seq)+ self._previous.move_to_end(key)+ self._stored += len(items)+ if self._lost.get(key, seq) < seq:+ # Newer than everything evicted: the baseline is whole again.+ del self._lost[key]++ def _evict(self, admitting: str) -> None:+ """Forget the least recent entries until the combined budget holds.++ Called with the lock held, right after admitting a request for+ ``admitting``, whose baseline is never a candidate. Each pass removes+ one entry until only that baseline and one pending request are left:+ each holds at most :data:`MAX_STORED_ITEMS` items, so they stay.+ """+ while self._stored + self._pending_items > MAX_STORED_ITEMS:+ oldest_ticket = next(iter(self._pending), None)+ oldest_key = next((k for k in self._previous if k != admitting), None)+ if oldest_key is None and len(self._pending) <= 1:+ return+ # On a tie the baseline goes: the request that used it is newer.+ if oldest_key is not None and (+ oldest_ticket is None or self._previous[oldest_key][2] <= oldest_ticket+ ):+ settled, evicted, _ = self._previous.pop(oldest_key)+ self._stored -= len(evicted)+ self._mark_lost(oldest_key, settled)+ else:+ ticket, (key, dropped) = self._pending.popitem(last=False)+ self._pending_items -= len(dropped)+ self._mark_lost(key, ticket)++ def _mark_lost(self, key: str, ticket: int) -> None:+ """Remember that ``key``'s items up to ``ticket`` were evicted (lock held)."""+ self._lost[key] = max(ticket, self._lost.get(key, 0))+ self._lost.move_to_end(key)+ while len(self._lost) > MAX_LOST_KEYS:+ forgotten, _ = self._lost.popitem(last=False)+ if not self._lost_overflowed:+ self._lost_overflowed = True+ logger.info(+ "savings: lost-baseline memory full (MAX_LOST_KEYS=%d); requests "+ "without a baseline now price at the cache-read floor",+ MAX_LOST_KEYS,+ )+ # Unmarked, a stale baseline would price old items as new: drop it,+ # so the conversation counts as lost under the overflow rule.+ stale = self._previous.pop(forgotten, None)+ if stale is not None:+ self._stored -= len(stale[1])+++_ledger = ItemLedger()+++def get_item_ledger() -> ItemLedger:+ """Process-wide ledger: one conversation spans many requests and handlers."""+ return _ledger+++def prefix_key(conversation: str | None, model: str, original: Sequence[Item]) -> str:+ """Which previous request a request is compared against.++ The conversation's explicit identity (``savings_conversation_key``) plus+ the tools and instructions: a side request in the same session (compaction,+ review) carries a different prefix and must not replace the conversation's+ previous request. Without an identity, the model plus the first input item+ stand in. Requests that collide on the key overwrite each other's previous+ request, shortening the shared prefix: more of the removal is then priced+ as new, at the miss rate, which overstates the saving.+ """+ head = "\x00".join(identity for identity, _ in original[:2])+ if conversation:+ # The model too: the provider's cache is per model, and so are counts.+ return _digest("\x00".join([conversation, model, head]))+ return _digest("\x00".join([model, *(identity for identity, _ in original[:3])]))+++def _input_cost(+ rates: CacheRates, *, uncached: float, read: float, write_5m: float, write_1h: float+) -> float:+ return (+ uncached * rates.uncached+ + read * rates.read+ + write_5m * rates.write_5m+ + write_1h * rates.write_1h+ )+++def _min_cacheable_tokens(model: str, provider: str | None) -> int:+ if provider == "openai":+ return OPENAI_MIN_CACHEABLE_TOKENS+ from headroom.pricing.counterfactual import catalog_min_cacheable_tokens++ return catalog_min_cacheable_tokens(model) or DEFAULT_MIN_CACHEABLE_TOKENS+++def _unpriced(measure: ItemMeasure, *, k: float, estimated: bool) -> SavingsRecord:+ return SavingsRecord(+ saved_usd=0.0,+ actual_input_cost_usd=0.0,+ counterfactual_input_cost_usd=0.0,+ removed_tokens=measure.removed,+ saved_tokens=k * measure.removed,+ k=k,+ warm=0.0,+ write_share=0.0,+ estimated=estimated,+ priced=False,+ )+++def compute_savings_record(+ measure: ItemMeasure,+ usage: CacheMix | None,+ *,+ model: str,+ provider: str | None = None,+ estimated: bool = False,+) -> SavingsRecord:+ """Price one request's removal as ``counterfactual_cost - actual_cost``.++ ``usage`` is the provider's billed input as a :class:`CacheMix` (normalized+ here, so an inferred OpenAI write counts as the uncached slice it is).+ ``None`` or an empty mix means the provider reported nothing (an aborted+ stream); the record then prices at list with ``k = 1`` and is marked+ ``estimated``.++ For a request the provider ran as several server-side iterations, pass the+ FIRST iteration's usage when it is available (``estimated=False``); the+ later iterations are a cost the caller books separately. Pass the summed+ usage with ``estimated=True`` only when per-iteration usage is not+ available: the sum describes more prompts than the one sent body, so+ ``k`` is not calibrated from it (``k = 1``) and only its cache ratios are+ used. ``provider`` must be given for OpenAI's 128-token cache blocks to+ apply.++ Never raises: a savings figure must not be able to fail a request.+ """+ try:+ record = _compute(measure, usage, model=model, provider=provider, estimated=estimated)+ # An approximate item pairing leaves the totals exact but not the+ # old/new split the price depends on.+ return record if measure.aligned else replace(record, estimated=True)+ except Exception as exc: # noqa: BLE001 - pricing must never fail a request+ logger.debug(+ "savings: record failed for model=%s: %s", safe_id(model), describe_exception(exc)+ )+ return _unpriced(measure, k=1.0, estimated=True)+++def _compute(+ measure: ItemMeasure,+ usage: CacheMix | None,+ *,+ model: str,+ provider: str | None,+ estimated: bool,+) -> SavingsRecord:+ from headroom.pricing.counterfactual import LONG_CONTEXT_THRESHOLD_TOKENS, resolve_rates++ def rates_for(billed: float, *, for_billing: bool = False) -> CacheRates | None:+ return resolve_rates(+ model,+ long_context=billed > LONG_CONTEXT_THRESHOLD_TOKENS,+ provider=provider,+ for_billing=for_billing,+ )++ removed = measure.removed+ # With the baseline lost, the new items may be old ones the provider had+ # cached. Their removal prices at the read rate, the floor of any input+ # token: the cache ratio and minimum below describe a prefix the ledger no+ # longer knows. Tokens Headroom added to new items stay at the miss rate.+ floor = measure.removed_new + measure.grown_new if measure.baseline_lost else 0+ mix = usage.normalized() if usage is not None else None+ if mix is None or mix.billed <= 0:+ actual_rates = rates_for(measure.sent_tokens)+ cf_rates = rates_for(measure.sent_tokens + removed)+ if actual_rates is None or cf_rates is None:+ return _unpriced(measure, k=1.0, estimated=True)+ actual = measure.sent_tokens * actual_rates.uncached+ counterfactual = (+ measure.sent_tokens + removed - floor+ ) * cf_rates.uncached + floor * cf_rates.read+ return SavingsRecord(+ saved_usd=counterfactual - actual,+ actual_input_cost_usd=actual,+ counterfactual_input_cost_usd=counterfactual,+ removed_tokens=removed,+ saved_tokens=float(removed),+ k=1.0,+ warm=0.0,+ write_share=0.0,+ estimated=True,+ )++ billed = mix.billed+ writes = mix.write_5m + mix.write_1h+ k = billed / measure.sent_tokens if measure.sent_tokens > 0 and not estimated else 1.0+ hit = mix.read / billed+ sigma = measure.sent_shared_prefix / measure.sent_tokens if measure.sent_tokens > 0 else 0.0+ warm = min(1.0, hit / sigma) if sigma > 0 else 0.0+ misses = mix.uncached + writes+ w = writes / misses if misses > 0 else 0.0++ # The counterfactual must obey its own cache minimum. The catalog's+ # minimum is used unless this request was demonstrably cached or written+ # below it; then the catalog is wrong and no minimum is applied, the+ # direction that cannot inflate the saving.+ min_cacheable = _min_cacheable_tokens(model, provider)+ proven = mix.read if mix.read > 0 else (billed if writes > 0 else min_cacheable)+ threshold = min_cacheable if proven >= min_cacheable else 0+ # Removed or busted old tokens could only have been reads if the+ # counterfactual's old prefix was itself cacheable.+ raw_warm = warm+ if k * (measure.sent_shared_prefix + measure.removed_old) < threshold:+ warm = 0.0++ # The counterfactual's buckets are built whole, not as deltas clamped onto+ # the actual ones: a counterfactual smaller than the request (an old item+ # Headroom grew) must not inherit reads or writes it was too small to make.+ # Its reads are the actual reads plus the removed/busted old tokens that+ # would have hit, minus the reads a grown old item contributed (those go+ # whatever the prefix size), never more than its whole billed input, and+ # none below its cache minimum. ``cf_billed`` is ``k * original`` when k is+ # calibrated; the clamp matters only for summed (estimated) usage.+ cf_billed = max(0.0, billed + k * removed)+ gained = k * (max(0, measure.removed_old) + measure.bust) * warm+ lost = k * min(0, measure.removed_old) * raw_warm+ cf_read = min(max(0.0, mix.read + gained + lost), cf_billed)+ if cf_read < threshold:+ # Never fires when nothing changed: the actual reads are 0 or proven+ # at least the threshold.+ cf_read = 0.0+ if provider == "openai" and cf_read != mix.read:+ # Hits come in whole 128-token blocks: round down, but a counterfactual+ # that adds reads keeps at least the actual ones. The ``!=`` keeps an+ # unchanged request exactly at the actual reads.+ blocks = (cf_read // OPENAI_CACHE_BLOCK_TOKENS) * OPENAI_CACHE_BLOCK_TOKENS+ cf_read = max(blocks, mix.read) if cf_read > mix.read else blocks+ if floor > 0:+ cf_read = min(cf_read + k * floor, cf_billed)+ # Everything else missed, written in the proportion the request wrote,+ # unless the counterfactual is smaller than anything shown cacheable: a+ # write it could not make would inflate the saving.+ cf_miss = cf_billed - cf_read+ cf_write = cf_miss * w if cf_billed >= min(min_cacheable, proven) else 0.0+ w5_share = mix.write_5m / writes if writes > 0 else 1.0+ cf_buckets = {+ "uncached": cf_miss - cf_write,+ "read": cf_read,+ "write_5m": cf_write * w5_share,+ "write_1h": cf_write * (1.0 - w5_share),+ }+ actual_buckets = {+ "uncached": float(mix.uncached),+ "read": float(mix.read),+ "write_5m": float(mix.write_5m),+ "write_1h": float(mix.write_1h),+ }++ # Tokens the request was billed for price at billing rates on both sides+ # (the counterfactual's at its own context tier); only the tokens a+ # counterfactual bucket holds beyond the actual one price at the+ # savings-estimate rates. Every priced quantity is nonnegative.+ actual_rates = rates_for(billed, for_billing=True)+ cf_billing_rates = rates_for(cf_billed, for_billing=True)+ cf_rates = rates_for(cf_billed)+ if actual_rates is None or cf_billing_rates is None or cf_rates is None:+ return _unpriced(measure, k=k, estimated=estimated)++ actual = _input_cost(actual_rates, **actual_buckets)+ counterfactual = _input_cost(+ cf_billing_rates, **{b: min(n, actual_buckets[b]) for b, n in cf_buckets.items()}+ ) + _input_cost(cf_rates, **{b: max(0.0, n - actual_buckets[b]) for b, n in cf_buckets.items()})+ return SavingsRecord(+ saved_usd=counterfactual - actual,+ actual_input_cost_usd=actual,+ counterfactual_input_cost_usd=counterfactual,+ removed_tokens=removed,+ saved_tokens=k * removed,+ k=k,+ warm=warm,+ write_share=w,+ estimated=estimated,+ )+++__all__ = [+ "Item",+ "ItemCounts",+ "ItemLedger",+ "ItemMeasure",+ "SavingsRecord",+ "compute_savings_record",+ "get_item_ledger",+ "measure_items",+ "prefix_key",+ "responses_items",+]diff --git a/tests/test_savings_items.py b/tests/test_savings_items.pynew file mode 100644index 000000000..f4166d421--- /dev/null+++ b/tests/test_savings_items.py@@ -0,0 +1,376 @@+"""Per-item original/sent measurement feeding the savings record."""++from __future__ import annotations++import json+import threading+from typing import Any++import httpx+import pytest+import respx+from fastapi.testclient import TestClient++from headroom.proxy import savings+from headroom.proxy.loopback_guard import require_loopback+from headroom.proxy.savings import ItemLedger, ItemMeasure, prefix_key, responses_items+from headroom.proxy.server import ProxyConfig, create_app+++def _words(text: str) -> int:+ return len(text.split(","))+++def _items(*texts: str) -> list[tuple[str, str]]:+ return [(t, t) for t in texts]+++def _turn(ledger, original, sent, count=_words, *, succeeded=True):+ """Measure one request and settle it, as the outcome funnel does."""+ m = ledger.measure("conv", original, sent, count)+ ledger.settle(m, succeeded=succeeded)+ return m+++def test_a_conversation_is_measured_against_its_previous_request():+ ledger = ItemLedger()+ calls: list[str] = []++ def count(text: str) -> int:+ calls.append(text)+ return _words(text)++ first = ['["t1","t2"]', '"sys"', '{"a":1,"b":2,"c":3}']+ sent1 = ['["t1"]', '"sys"', '{"a":1,"b":2,"c":3}']+ second = [*first, '{"d":4,"e":5}']+ sent2 = [*sent1, '{"d":4}']++ assert _turn(ledger, _items(*first), _items(*sent1), count) == ItemMeasure(+ sent_tokens=5, removed_old=0, removed_new=1, sent_shared_prefix=0, bust=0+ )+ assert _turn(ledger, _items(*second), _items(*sent2), count) == ItemMeasure(+ sent_tokens=6, removed_old=1, removed_new=1, sent_shared_prefix=5, bust=0+ )+ # Re-sent items are tokenized once, not once per turn.+ assert len(calls) == len(set(calls))+++def test_items_follow_the_providers_prompt_order():+ body = {"tools": [{"name": "t"}], "instructions": "be brief", "input": [{"x": 1}, {"y": 2}]}+ assert responses_items(body) == _items('[{"name":"t"}]', '"be brief"', '{"x":1}', '{"y":2}')+ assert responses_items({"input": "hi"}) == _items("[]", '""', '"hi"')+++def test_reasoning_ciphertext_is_identity_not_billed_text():+ """Codex replays encrypted reasoning every turn; counting the ciphertext+ made Headroom's count drift far above the provider's bill."""+ item = {"type": "reasoning", "summary": [], "encrypted_content": "gAAAA" * 50}+ [(identity, billable)] = responses_items({"input": [item]})[2:]+ assert "gAAAA" in identity+ assert billable == '{"type":"reasoning","summary":[]}'+++def test_a_folded_transcript_keeps_the_old_items_old():+ """A hook folding two items into one shifts every later position; pairing+ by position then priced untouched new items as removed."""+ ledger = ItemLedger()+ t, i = "[]", '"sys"'+ a, b, c, d = ("," * 9 + x for x in "abcd") # 10 tokens each under _words+ _turn(ledger, _items(t, i, a, b, c), _items(t, i, a, b, c), _words)+ m = _turn(ledger, _items(t, i, a, b, c, d), _items(t, i, "f", c, d), _words)+ assert (m.removed_old, m.removed_new) == (19, 0)+ # The fold moved c: its sent copy is outside the shared prefix, a bust.+ assert m.bust == 11 and m.aligned is False+++def test_a_fold_after_an_item_compressed_in_place_keeps_the_prefix():+ """The common Codex case: an early tool output compressed every turn, then+ a hook folds the new items. The compressed item must still pair at the head,+ or every later item is priced as removed and as a bust."""+ ledger = ItemLedger()+ t, i = "[]", '"sys"'+ big, small = "," * 99 + "x", "x" # 100 and 1 tokens under _words+ n1, n2 = "," * 9 + "n", "," * 9 + "m"+ _turn(ledger, _items(t, i, big), _items(t, i, small), _words)+ m = _turn(ledger, _items(t, i, big, n1, n2), _items(t, i, small, "f"), _words)+ assert (m.sent_shared_prefix, m.bust) == (3, 0)+ assert (m.removed_old, m.removed_new) == (99, 19)+ assert m.aligned is False+++def test_an_approximate_pairing_marks_the_record_estimated():+ m = ItemMeasure(+ sent_tokens=10, removed_old=0, removed_new=5, sent_shared_prefix=0, bust=0, aligned=False+ )+ assert savings.compute_savings_record(m, None, model="unpriced-model").estimated+++def test_a_failed_request_does_not_become_the_baseline():+ ledger = ItemLedger()+ first = _items("[]", '"sys"', '"a"')+ _turn(ledger, first, first)+ _turn(ledger, _items("[]", '"other"', '"b"'), _items("[]", '"other"', '"b"'), succeeded=False)+ assert _turn(ledger, first, first).sent_shared_prefix == 3+++def test_a_late_settling_older_turn_keeps_the_newer_baseline():+ ledger = ItemLedger()+ old, new = _items("[]", '"a"'), _items("[]", '"a"', '"b"')+ slow = ledger.measure("conv", old, old, _words)+ fast = ledger.measure("conv", new, new, _words)+ ledger.settle(fast, succeeded=True)+ ledger.settle(slow, succeeded=True)+ assert _turn(ledger, new, new).sent_shared_prefix == 3+++def test_requests_awaiting_their_outcome_are_bounded_by_items(monkeypatch):+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ abandoned = ledger.measure("a", _items("[]", '"x"', '"y"'), _items("[]", '"x"', '"y"'), _words)+ kept = ledger.measure("b", _items("[]", '"z"'), _items("[]", '"z"'), _words)+ # The oldest pending request was forgotten; settling it is a no-op.+ ledger.settle(abandoned, succeeded=True)+ ledger.settle(kept, succeeded=True)+ # Each probe is itself a pending request under the budget: kept first.+ assert (+ ledger.measure("b", _items("[]", '"z"'), _items("[]", '"z"'), _words).sent_shared_prefix+ == 2+ )+ again = _items("[]", '"x"', '"y"')+ assert ledger.measure("a", again, again, _words).sent_shared_prefix == 0+++def test_a_slow_older_measure_never_evicts_a_newer_one(monkeypatch):+ """JerrettDavis's #4240 interleaving: ticket 1 stalls inside token+ counting while ticket 2 measures four items. Eviction must drop the older+ ticket, not the newer one that finished first."""+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ counting, release = threading.Event(), threading.Event()++ def stalled(text: str) -> int:+ counting.set()+ assert release.wait(timeout=10)+ return _words(text)++ older: list[ItemMeasure] = []+ slow = _items("[]", '"x"')+ thread = threading.Thread(target=lambda: older.append(ledger.measure("a", slow, slow, stalled)))+ thread.start()+ assert counting.wait(timeout=10)+ new = _items("[]", '"b1"', '"b2"', '"b3"')+ newer = ledger.measure("b", new, new, _words)+ release.set()+ thread.join(timeout=10)++ ledger.settle(newer, succeeded=True)+ ledger.settle(older[0], succeeded=True)+ assert ledger.measure("b", new, new, _words).sent_shared_prefix == 4+++def test_settled_and_pending_items_share_one_budget(monkeypatch):+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ idle = _items("[]", '"a1"', '"a2"', '"a3"')+ _turn(ledger, idle, idle)+ active = _items("[]", '"b1"', '"b2"', '"b3"')+ in_flight = ledger.measure("b", active, active, _words)+ # Admitting the in-flight request evicted the idle baseline at once.+ assert ledger._stored + ledger._pending_items == 4+ ledger.settle(in_flight, succeeded=True)+ assert ledger.measure("b", active, active, _words).sent_shared_prefix == 4+ assert ledger.measure("conv", idle, idle, _words).sent_shared_prefix == 0+++def test_a_request_larger_than_the_budget_is_kept_up_to_the_budget(monkeypatch):+ """Keeping nothing would leave the next request with no baseline. The+ first items are kept instead; a request measured against them may have+ old items past the cut, so its baseline counts as lost. Each turn still+ replaces the baseline, however large the conversation."""+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ big = _items("[]", '"s"', '"u1"', '"u2"', '"u3"')+ m = _turn(ledger, big, big)+ assert (m.aligned, m.baseline_lost) == (True, False)+ longer = [*big, *_items('"u4"')]+ m = _turn(ledger, longer, longer)+ assert (m.sent_shared_prefix, m.aligned, m.baseline_lost) == (4, False, True)+ assert not ledger._lost # the cut-off rule fired, nothing was evicted+ assert ledger._stored + ledger._pending_items <= 4+++def test_a_failed_turn_keeps_its_conversations_baseline(monkeypatch):+ """Devin's #4240 example: admitting a four-item turn under a four-item+ budget evicted the conversation's own baseline; the turn then got a 429,+ and the next identical request saw no shared prefix at all."""+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ turn = _items("[]", '"s"', '"u1"', '"u2"')+ _turn(ledger, turn, turn)+ _turn(ledger, turn, turn, succeeded=False)+ assert _turn(ledger, turn, turn).sent_shared_prefix == 4+++def test_overlapping_turns_keep_the_baseline_until_one_succeeds(monkeypatch):+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 8)+ ledger = ItemLedger()+ base = _items("[]", '"s"', '"u1"')+ _turn(ledger, base, base)+ first = [*base, *_items('"u2"')]+ second = [*first, *_items('"u3"', '"u4"')]+ m1 = ledger.measure("conv", first, first, _words)+ # Three baseline items plus ten pending: the older turn goes, not the baseline.+ m2 = ledger.measure("conv", second, second, _words)+ ledger.settle(m2, succeeded=False)+ ledger.settle(m1, succeeded=False)+ m3 = ledger.measure("conv", first, first, _words)+ # Still the old baseline, but the forgotten turn may have been cached.+ assert (m3.sent_shared_prefix, m3.baseline_lost, m3.aligned) == (3, True, False)+ ledger.settle(m3, succeeded=True)+ m4 = _turn(ledger, second, second)+ assert (m4.sent_shared_prefix, m4.baseline_lost, m4.aligned) == (4, False, True)+++def _probe(ledger, key, items):+ m = ledger.measure(key, items, items, _words)+ ledger.settle(m, succeeded=True)+ return m+++def test_a_lost_baseline_is_flagged_and_a_new_conversation_is_not(monkeypatch):+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ ledger = ItemLedger()+ a, b, c = (_items("[]", f'"{n}1"', f'"{n}2"') for n in "abc")+ assert not _probe(ledger, "a", a).baseline_lost+ # Admitting b evicts a's baseline: another conversation, the least recent.+ assert not _probe(ledger, "b", b).baseline_lost+ again = _probe(ledger, "a", a)+ assert (again.sent_shared_prefix, again.baseline_lost, again.aligned) == (0, True, False)+ assert not _probe(ledger, "c", c).baseline_lost+++def test_past_the_loss_bound_every_missing_baseline_counts_as_lost(monkeypatch):+ """Forgetting a loss must not let its conversation pass for a new one."""+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ monkeypatch.setattr(savings, "MAX_LOST_KEYS", 1)+ ledger = ItemLedger()+ a, b, c, d = (_items("[]", f'"{n}1"', f'"{n}2"') for n in "abcd")+ _probe(ledger, "a", a)+ _probe(ledger, "b", b) # loses a+ _probe(ledger, "c", c) # loses b; a's loss is forgotten+ assert _probe(ledger, "a", a).baseline_lost+ assert _probe(ledger, "d", d).baseline_lost+++def test_a_forgotten_loss_drops_the_stale_baseline_it_leaves(monkeypatch):+ """a's newer request was evicted while its baseline survived. Once that+ loss is forgotten, the stale baseline must not pass for a whole one, or+ the evicted request's cached items would price as new."""+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)+ monkeypatch.setattr(savings, "MAX_LOST_KEYS", 1)+ ledger = ItemLedger()+ one, two = _items('"x"'), _items("[]", '"y"')+ _probe(ledger, "x", one)+ _probe(ledger, "a", one)+ a1 = ledger.measure("a", one, one, _words)+ ledger.settle(ledger.measure("x", one, one, _words), succeeded=False) # x used after a1+ a2 = ledger.measure("a", one, one, _words)+ ledger.measure("b", one, one, _words) # evicts a1: a is lost, its baseline stays+ ledger.settle(a1, succeeded=True)+ ledger.settle(a2, succeeded=False)+ assert "a" in ledger._lost+ ledger.measure("c", two, two, _words) # evicts x's baseline: a's loss is forgotten+ m = ledger.measure("a", one, one, _words)+ assert (m.baseline_lost, m.aligned) == (True, False)+++def test_a_measure_that_fails_releases_its_slot():+ """The caller logs a failed measure and never settles it."""+ ledger = ItemLedger()++ def broken(text: str) -> int:+ raise ValueError("tokenizer failed")++ for _ in range(3):+ with pytest.raises(ValueError):+ ledger.measure("conv", _items("[]"), _items("[]"), broken)+ assert not ledger._pending+++def test_messages_stand_in_for_input():+ assert responses_items({"messages": [{"role": "user", "content": "hi"}]})[2:] == _items(+ '{"role":"user","content":"hi"}'+ )+++def test_side_requests_do_not_replace_the_conversations_previous_request():+ main = _items("[]", '"agent"', '"hi"')+ side = _items("[]", '"summarize"', '"hi"')+ assert prefix_key("conv-1", "m", main) != prefix_key("conv-1", "m", side)+ assert prefix_key("conv-1", "m", main) == prefix_key(+ "conv-1", "m", [*main, ('"more"', '"more"')]+ )+ assert prefix_key(None, "m", main) == prefix_key(None, "m", [*main, ('"more"', '"more"')])+ assert prefix_key(None, "m", main) != prefix_key(None, "other", main)+++def _sse(usage: dict) -> str:+ done = {"type": "response.completed", "response": {"id": "r", "output": [], "usage": usage}}+ return f"event: response.completed\ndata: {json.dumps(done)}\n\n"++[email protected]("stream", [False, True])+def test_responses_outcomes_carry_the_measure(monkeypatch, stream):+ app = create_app(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ )+ )+ app.dependency_overrides[require_loopback] = lambda: None+ outcomes: list[Any] = []++ # Capture at the funnel's first effect, so the real funnel (which settles+ # the measure) still runs.+ monkeypatch.setattr("headroom.telemetry.session.record_outcome", outcomes.append)+ monkeypatch.setattr(savings, "_ledger", ItemLedger())+ usage = {"input_tokens": 10, "output_tokens": 1}+ respx.post("https://api.openai.com/v1/responses").mock(+ return_value=httpx.Response(+ 200,+ text=_sse(usage),+ headers={"content-type": "text/event-stream"},+ )+ if stream+ else httpx.Response(200, json={"id": "r", "output": [], "usage": usage})+ )+ turn1 = [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "a"}]}]+ turn2 = [+ *turn1,+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "b"}]},+ ]+ with TestClient(app) as client:+ for items in (turn1, turn2):+ r = client.post(+ "/v1/responses",+ json={+ "model": "gpt-4o",+ "instructions": "be brief",+ "input": items,+ "stream": stream,+ "client_metadata": {"session_id": "conv-test"},+ },+ headers={"authorization": "Bearer sk-test"},+ )+ assert r.status_code == 200+ r.read()++ first, second = (o.savings_measure for o in outcomes)+ assert first.removed == 0 and first.sent_shared_prefix == 0+ # Nothing was compressed, so the whole first request is the shared prefix.+ assert second.sent_shared_prefix == first.sent_tokens+ assert second.removed == 0 and second.bust == 0diff --git a/tests/test_savings_record.py b/tests/test_savings_record.pynew file mode 100644index 000000000..b371522a4--- /dev/null+++ b/tests/test_savings_record.py@@ -0,0 +1,368 @@+"""The per-request savings record: counterfactual cost minus actual cost."""++from __future__ import annotations++import dataclasses++import pytest++from headroom.pricing import counterfactual+from headroom.pricing.counterfactual import CacheMix, CacheRates+from headroom.proxy.savings import (+ ItemCounts,+ ItemMeasure,+ compute_savings_record,+ measure_items,+)++# $3/M input, $0.30/M cache read, $3.75/M 5m write, $6/M 1h write; the+# long-context tier doubles everything.+IN, READ, WRITE, WRITE_1H = 3e-6, 3e-7, 3.75e-6, 6e-6+BASE = CacheRates(read=READ, write_5m=WRITE, write_1h=WRITE_1H, uncached=IN)+LONG = CacheRates(read=2 * READ, write_5m=2 * WRITE, write_1h=2 * WRITE_1H, uncached=2 * IN)++[email protected](autouse=True)+def _fixed_catalog(monkeypatch):+ """Pin the catalog so the arithmetic is tested whatever litellm ships."""++ def rates(model, *, long_context=False, provider=None, for_billing=False):+ return None if model == "unpriced" else (LONG if long_context else BASE)++ monkeypatch.setattr(counterfactual, "resolve_rates", rates)+ monkeypatch.setattr(+ counterfactual,+ "catalog_min_cacheable_tokens",+ lambda model: 4096 if model == "haiku" else 1024,+ )+++def _formula(m: ItemMeasure, u: CacheMix) -> float:+ """The closed form the record must agree with."""+ billed = u.read + u.write_5m + u.uncached+ k = billed / m.sent_tokens+ warm = min(1.0, (u.read / billed) / (m.sent_shared_prefix / m.sent_tokens))+ w = u.write_5m / (u.uncached + u.write_5m)+ p_miss = w * WRITE + (1 - w) * IN+ return (+ k * m.removed_old * (warm * READ + (1 - warm) * p_miss)+ + k * m.removed_new * p_miss+ - k * m.bust * warm * (p_miss - READ)+ )+++WARM = CacheMix(uncached=2_000, read=90_000, write_5m=8_000)+MEASURE = ItemMeasure(+ sent_tokens=95_000, removed_old=50_000, removed_new=10_000, sent_shared_prefix=88_000, bust=0+)+++def test_worked_example():+ record = compute_savings_record(MEASURE, WARM, model="sonnet", provider="anthropic")+ assert record.k == pytest.approx(1.0526, abs=1e-4)+ assert record.warm == pytest.approx(0.97159, abs=1e-5)+ assert record.write_share == pytest.approx(0.8)+ assert record.saved_usd == pytest.approx(0.058618, abs=1e-6)+ assert record.saved_usd == pytest.approx(_formula(MEASURE, WARM), abs=1e-9)+ assert record.saved_usd == pytest.approx(+ record.counterfactual_input_cost_usd - record.actual_input_cost_usd+ )+ assert record.saved_tokens == pytest.approx(100_000 / 95_000 * 60_000)+ assert not record.estimated+++def test_a_cache_bust_headroom_caused_is_charged_against_it():+ busted = dataclasses.replace(MEASURE, bust=5_000)+ record = compute_savings_record(busted, WARM, model="sonnet", provider="anthropic")+ assert record.saved_usd == pytest.approx(_formula(busted, WARM), abs=1e-9)+ assert record.saved_usd == pytest.approx(0.058618 - 0.016875, abs=1e-6)+++def test_a_bust_never_claims_more_reads_than_the_request_missed():+ """Item-level P_S undercounts a token-level hit, so hit > sigma clips warm+ to 1; the bust charge must still not drive the counterfactual negative."""+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=0, removed_new=0, sent_shared_prefix=2_000, bust=8_000+ )+ usage = CacheMix(uncached=500, read=9_000, write_5m=500)+ record = compute_savings_record(measure, usage, model="sonnet", provider="anthropic")+ assert record.counterfactual_input_cost_usd >= 0+ # At most the 1,000 missed tokens could have been reads instead.+ assert record.saved_usd == pytest.approx(-(500 * (IN - READ) + 500 * (WRITE - READ)))+++def test_an_old_item_headroom_grew_inherits_no_reads_below_the_cache_minimum():+ """JerrettDavis's #4236 case: a 1,000-token item sent as 10,000 tokens on+ successive requests, prompt cached. The 1,000-token counterfactual is+ under the 1,024-token minimum, so none of it could have been a read."""+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0+ )+ usage = CacheMix(read=9_000, uncached=1_000)+ for provider in ("openai", "anthropic"):+ record = compute_savings_record(measure, usage, model="gpt", provider=provider)+ assert record.actual_input_cost_usd == pytest.approx(0.0057)+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN) # $0.003+ assert record.saved_usd == pytest.approx(-0.0027)+++def test_summed_iteration_usage_does_not_calibrate_k():+ """Usage summed over server-side iterations describes more prompts than+ the one sent body, so it must not scale the removal."""+ summed = CacheMix(read=270_000, uncached=6_000, write_5m=24_000)+ record = compute_savings_record(+ MEASURE, summed, model="sonnet", provider="anthropic", estimated=True+ )+ assert record.k == 1.0+ assert record.estimated+ assert record.saved_tokens == 60_000+++def test_a_cold_cache_prices_every_removed_token_as_a_miss():+ cold = CacheMix(uncached=10_000, read=0, write_5m=90_000)+ record = compute_savings_record(MEASURE, cold, model="sonnet", provider="anthropic")+ assert record.warm == 0.0+ p_miss = 0.9 * WRITE + 0.1 * IN+ assert record.saved_usd == pytest.approx(100_000 / 95_000 * 60_000 * p_miss)+++def test_one_hour_writes_price_at_their_own_rate():+ usage = CacheMix(uncached=2_000, read=90_000, write_1h=8_000)+ record = compute_savings_record(MEASURE, usage, model="sonnet", provider="anthropic")+ k, warm = 100_000 / 95_000, record.warm+ p_miss = 0.8 * WRITE_1H + 0.2 * IN+ expected = k * 50_000 * (warm * READ + (1 - warm) * p_miss) + k * 10_000 * p_miss+ assert record.saved_usd == pytest.approx(expected)+++def test_an_inferred_openai_write_is_the_uncached_slice_not_a_write():+ usage = CacheMix(read=90_000, write_total=10_000, uncached=10_000, inferred_write=True)+ record = compute_savings_record(MEASURE, usage, model="gpt", provider="openai")+ assert record.k == pytest.approx(100_000 / 95_000)+ assert record.write_share == 0.0+++def test_openai_reads_come_in_128_token_blocks():+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=1_000, removed_new=0, sent_shared_prefix=10_000, bust=0+ )+ usage = CacheMix(read=9_984, uncached=16) # 78 blocks+ record = compute_savings_record(measure, usage, model="gpt", provider="openai")+ # 1,000 * k removed reads would end mid-block: only whole blocks count.+ total_read = 9_984 + 1_000 * record.k+ reads = (total_read // 128) * 128 - 9_984+ assert record.saved_usd == pytest.approx(reads * READ + (1_000 * record.k - reads) * IN)+++def test_a_counterfactual_prefix_below_the_models_cache_minimum_is_never_warm():+ # A 3,000-token old prefix: cacheable on Sonnet (1,024), not on Haiku (4,096).+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=1_000, removed_new=0, sent_shared_prefix=2_000, bust=0+ )+ usage = CacheMix(uncached=5_000, read=5_000)+ assert compute_savings_record(measure, usage, model="sonnet").warm > 0+ assert compute_savings_record(measure, usage, model="haiku").warm == 0.0+++def test_the_counterfactual_is_priced_at_its_own_context_tier():+ measure = ItemMeasure(+ sent_tokens=150_000, removed_old=0, removed_new=100_000, sent_shared_prefix=0, bust=0+ )+ record = compute_savings_record(measure, CacheMix(uncached=150_000), model="sonnet")+ assert record.actual_input_cost_usd == pytest.approx(150_000 * IN)+ assert record.counterfactual_input_cost_usd == pytest.approx(250_000 * 2 * IN)+++def test_missing_usage_prices_at_list_per_tier_and_says_so():+ measure = dataclasses.replace(MEASURE, sent_tokens=150_000, removed_new=50_000)+ record = compute_savings_record(measure, None, model="sonnet")+ assert record.estimated+ assert record.actual_input_cost_usd == pytest.approx(150_000 * IN)+ assert record.counterfactual_input_cost_usd == pytest.approx(250_000 * 2 * IN)+++def test_an_unpriced_model_reports_tokens_but_no_dollars():+ record = compute_savings_record(MEASURE, WARM, model="unpriced")+ assert not record.priced+ assert record.saved_usd == 0.0+ assert record.removed_tokens == 60_000+++def test_items_split_into_old_and_new_against_the_previous_request():+ def item(name: str, original: int, sent: int, sent_name: str | None = None) -> ItemCounts:+ return ItemCounts(name, original, sent_name or name + "'", sent)++ first = [item("tools", 1_000, 200), item("sys", 500, 500, "sys"), item("u1", 300, 300, "u1")]+ second = [+ item("tools", 1_000, 200),+ item("sys", 500, 500, "sys"),+ # Headroom compressed this item differently this time: its sent copy+ # changed, so the provider missed it although the client resent it.+ item("u1", 300, 250, "u1-recompressed"),+ item("t1", 4_000, 1_000),+ # An item Headroom grew counts against the removal.+ item("t2", 100, 160),+ ]+ assert measure_items(first, None) == ItemMeasure(+ sent_tokens=1_000, removed_old=0, removed_new=800, sent_shared_prefix=0, bust=0+ )+ assert measure_items(second, first) == ItemMeasure(+ sent_tokens=2_110,+ removed_old=800 + 50,+ removed_new=3_000 - 60,+ sent_shared_prefix=700,+ bust=250,+ grown_new=60,+ )+++def test_billed_buckets_price_at_billing_rates(monkeypatch):+ """A catalog row without a cache-read rate bills reads via litellm's+ calculator (``for_billing``); only the changed tokens use the estimate."""+ billing = dataclasses.replace(BASE, read=5e-7)++ def rates(model, *, long_context=False, provider=None, for_billing=False):+ return billing if for_billing else BASE++ monkeypatch.setattr(counterfactual, "resolve_rates", rates)+ record = compute_savings_record(MEASURE, WARM, model="sonnet", provider="anthropic")+ assert record.actual_input_cost_usd == pytest.approx(2_000 * IN + 90_000 * 5e-7 + 8_000 * WRITE)+ assert record.saved_usd == pytest.approx(0.058618, abs=1e-6)+++def _fully_cached_original_of(original: int, *, model: str, provider: str):+ """A 10,000-token sent body read wholly from cache, whose original was+ ``original`` tokens: the counterfactual is that original alone."""+ measure = ItemMeasure(+ sent_tokens=10_000,+ removed_old=original - 10_000,+ removed_new=0,+ sent_shared_prefix=10_000,+ bust=0,+ )+ return compute_savings_record(measure, CacheMix(read=10_000), model=model, provider=provider)+++ ("original", "reads"),+ [(1_023, 0), (1_024, 1_024), (1_151, 1_024), (1_152, 1_152), (2_000, 1_920)],+)+def test_an_openai_counterfactual_reads_whole_blocks_above_its_own_minimum(original, reads):+ record = _fully_cached_original_of(original, model="gpt", provider="openai")+ expected = reads * READ + (original - reads) * IN+ assert record.counterfactual_input_cost_usd == pytest.approx(expected)++[email protected](("original", "reads"), [(4_095, 0), (4_096, 4_096), (4_097, 4_097)])+def test_an_anthropic_counterfactual_obeys_its_models_cache_minimum(original, reads):+ record = _fully_cached_original_of(original, model="haiku", provider="anthropic")+ expected = reads * READ + (original - reads) * IN+ assert record.counterfactual_input_cost_usd == pytest.approx(expected)+++def test_a_counterfactual_too_small_to_cache_pays_no_write_premium():+ """Below its minimum the counterfactual can be neither read nor written."""+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0+ )+ usage = CacheMix(read=9_000, write_5m=1_000)+ record = compute_savings_record(measure, usage, model="sonnet", provider="anthropic")+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN)+++def test_a_grown_old_item_takes_its_reads_out_even_with_new_items():+ """The new items keep the counterfactual above the minimum overall; its+ 1,000-token old prefix is still too small to have been read."""+ measure = ItemMeasure(+ sent_tokens=30_000,+ removed_old=-9_000,+ removed_new=20_000,+ sent_shared_prefix=10_000,+ bust=0,+ )+ usage = CacheMix(read=9_000, uncached=21_000)+ record = compute_savings_record(measure, usage, model="gpt", provider="openai")+ assert record.counterfactual_input_cost_usd == pytest.approx(41_000 * IN)+++def test_a_catalog_minimum_the_provider_disproved_is_not_applied():+ """Haiku's catalog says 4,096, but the provider read 3,000: keeping the+ counterfactual's 1,500 reads is the conservative price."""+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=-5_000, removed_new=0, sent_shared_prefix=10_000, bust=0+ )+ usage = CacheMix(read=3_000, uncached=7_000)+ record = compute_savings_record(measure, usage, model="haiku", provider="anthropic")+ assert record.counterfactual_input_cost_usd == pytest.approx(1_500 * READ + 3_500 * IN)+++def test_nothing_removed_saves_nothing_whatever_the_read_alignment():+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=0, removed_new=0, sent_shared_prefix=2_000, bust=0+ )+ for provider, usage in (+ ("openai", CacheMix(read=9_000, uncached=1_000)),+ ("anthropic", CacheMix(read=900, write_5m=9_100)),+ ):+ record = compute_savings_record(measure, usage, model="haiku", provider=provider)+ assert record.saved_usd == pytest.approx(0.0, abs=1e-12)+++def test_a_shrunk_bucket_is_never_priced_below_zero(monkeypatch):+ """Reads a grown item gave up price at the billing rate they were billed+ at, not at the estimate rate, so the counterfactual cannot go negative."""+ billing = dataclasses.replace(BASE, read=1.5e-6)++ def rates(model, *, long_context=False, provider=None, for_billing=False):+ return billing if for_billing else BASE++ monkeypatch.setattr(counterfactual, "resolve_rates", rates)+ measure = ItemMeasure(+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0+ )+ record = compute_savings_record(+ measure, CacheMix(read=9_000, uncached=1_000), model="gpt", provider="openai"+ )+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN)+++def test_a_disproved_minimum_still_writes_nothing_smaller_than_was_cached():+ """Haiku's catalog minimum is disproved by a 2,000-token read; a 500-token+ counterfactual is below anything shown cacheable, so it writes nothing."""+ measure = ItemMeasure(+ sent_tokens=3_000, removed_old=-2_500, removed_new=0, sent_shared_prefix=3_000, bust=0+ )+ usage = CacheMix(read=2_000, write_5m=1_000)+ record = compute_savings_record(measure, usage, model="haiku", provider="anthropic")+ reads = 2_000 + -2_500 * (2 / 3)+ assert record.counterfactual_input_cost_usd == pytest.approx(reads * READ + (500 - reads) * IN)+++def test_a_lost_baseline_prices_the_new_removal_at_the_read_floor():+ """With the conversation's baseline evicted, the new items may be old ones+ the provider had cached. Pricing them as new, at the miss rate, booked+ $0.2236 here; the read rate, the floor of any input token, books $0.0152.+ The 1,000 tokens Headroom added to a new item still cost the miss rate."""+ lost = ItemMeasure(+ sent_tokens=95_000,+ removed_old=0,+ removed_new=60_000 - 1_000,+ sent_shared_prefix=0,+ bust=0,+ aligned=False,+ baseline_lost=True,+ grown_new=1_000,+ )+ k, p_miss = 100_000 / 95_000, 0.8 * WRITE + 0.2 * IN+ as_new = compute_savings_record(+ dataclasses.replace(lost, baseline_lost=False), WARM, model="sonnet", provider="anthropic"+ )+ assert as_new.saved_usd == pytest.approx(k * 59_000 * p_miss) # $0.2236++ record = compute_savings_record(lost, WARM, model="sonnet", provider="anthropic")+ assert record.saved_usd == pytest.approx(k * 60_000 * READ - k * 1_000 * p_miss) # $0.0152+ assert record.saved_usd == pytest.approx(0.015158, abs=1e-6)+ assert record.estimated+ # Without usage too: list price for the rest, the read rate for the removal.+ unbilled = compute_savings_record(lost, None, model="sonnet", provider="anthropic")+ assert unbilled.saved_usd == pytest.approx(60_000 * READ - 1_000 * IN)