Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(savings): add the per-request counterfactual savings record (#4236)

Tejas Chopra2912b7b620d7
diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.py
index 2763120fb..5f3bc65cf 100644
--- a/headroom/pricing/counterfactual.py
+++ b/headroom/pricing/counterfactual.py
@@ -374,6 +374,26 @@ def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context:
return rate if math.isfinite(rate) and rate >= 0 else 0.0
+def _catalog_row(model: str) -> tuple[Any, str, dict[str, Any]] | None:
+ """``(litellm, resolved model, catalog row)``, or ``None``. Never raises."""
+ litellm = _litellm()
+ if litellm is None:
+ return None
+ try:
+ from headroom.pricing.litellm_pricing import resolve_litellm_model
+
+ resolved_model = resolve_litellm_model(model)
+ info = litellm.model_cost.get(resolved_model, {}) or {}
+ except Exception as exc:
+ logger.debug(
+ "counterfactual: LiteLLM catalog lookup failed for %s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
+ return None
+ return litellm, resolved_model, info
+
+
@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)
def resolve_rates(
model: str,
@@ -413,22 +433,10 @@ def resolve_rates(
catalog between cases must call ``resolve_rates.cache_clear()`` in between,
or the previous case's rates leak into the next.
"""
- litellm = _litellm()
- if litellm is None:
- return None
-
- try:
- from headroom.pricing.litellm_pricing import resolve_litellm_model
-
- resolved_model = resolve_litellm_model(model)
- info = litellm.model_cost.get(resolved_model, {}) or {}
- except Exception as exc:
- logger.debug(
- "counterfactual: LiteLLM catalog lookup failed for %s: %s",
- safe_id(model),
- describe_exception(exc),
- )
+ row = _catalog_row(model)
+ if row is None:
return None
+ litellm, resolved_model, info = row
base = info.get("input_cost_per_token")
# `is None` distinguishes "unknown model" from "genuinely free". `not base`
@@ -530,6 +538,19 @@ def resolve_rates(
)
+def catalog_min_cacheable_tokens(model: str) -> int | None:
+ """The catalog's smallest prompt the provider will cache for ``model``.
+
+ LiteLLM publishes it as ``prompt_cache_min_tokens`` (Anthropic: 1024 for
+ Sonnet, 4096 for Haiku 4.5). ``None`` when litellm or the row lacks it.
+ """
+ row = _catalog_row(model)
+ value = row[2].get("prompt_cache_min_tokens") if row is not None else None
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ return None
+ return int(value) if math.isfinite(value) and value > 0 else None
+
+
@dataclass(frozen=True)
class PricedSavings:
"""What a set of counterfactual tokens was worth, and how sure we are.
@@ -641,6 +662,7 @@ __all__ = [
"PricedSavings",
"Region",
"TokenSplit",
+ "catalog_min_cacheable_tokens",
"price_savings",
"resolve_rates",
"split_tokens",
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 4b5d299bc..3b1f62834 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -45,6 +45,7 @@ from headroom.proxy.provider_usage import (
usage_int,
)
from headroom.proxy.rate_limit_identity import rate_limit_identity
+from headroom.proxy.savings import ItemMeasure, get_item_ledger, prefix_key, responses_items
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
from headroom.proxy.tool_schema_savings_policy import (
@@ -6391,6 +6392,36 @@ class OpenAIHandlerMixin:
"upstream request for server-side retrieval handling"
)
+ # Per-item original/sent counts against this conversation's previous
+ # request, in Headroom's tokenizer (headroom.proxy.savings). ORIGINAL
+ # is the client's bytes, before memory, compression or shaping; SENT is
+ # the body forwarded below. All of it runs off the event loop: it
+ # decodes, serializes and tokenizes the whole transcript. Skipped for
+ # incremental bodies (previous_response_id / conversation): the
+ # provider bills context they do not carry. It covers the first
+ # upstream call only; continuations and re-drives are the wiring's to
+ # mark as estimated. The outcome funnel settles it (ItemLedger.settle).
+ savings_measure = None
+ if not (body.get("previous_response_id") or body.get("conversation")):
+
+ def _measure_items() -> ItemMeasure:
+ original = responses_items(json.loads(original_body_bytes))
+ return get_item_ledger().measure(
+ prefix_key(_pre_compression_conversation_key, model, original),
+ original,
+ responses_items(body),
+ tokenizer.count_text,
+ )
+
+ try:
+ savings_measure = await asyncio.to_thread(_measure_items)
+ except Exception as exc: # noqa: BLE001 - telemetry must never break a request
+ logger.warning(
+ "[%s] savings item measurement skipped: %s",
+ request_id,
+ describe_exception(exc),
+ )
+
try:
if stream and not buffered_stream_ccr:
# Streaming for Responses API uses semantic events
@@ -6415,6 +6446,7 @@ class OpenAIHandlerMixin:
waste_signals=waste_signals_dict,
conversation_key=responses_conversation_key,
conversation_tokens_saved=tokens_saved,
+ savings_measure=savings_measure,
)
else:
@@ -6883,6 +6915,7 @@ class OpenAIHandlerMixin:
tags=_resp_log_tags,
conversation_key=responses_conversation_key,
conversation_tokens_saved=tokens_saved,
+ savings_measure=savings_measure,
turn_id=compute_turn_id(model, body.get("instructions"), messages),
request_messages=messages
if getattr(self.config, "log_full_messages", False)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index c8de2c75f..f7dde7aca 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -32,6 +32,8 @@ from headroom.proxy.token_counting import gemini_output_tokens
if TYPE_CHECKING:
from fastapi.responses import Response, StreamingResponse
+ from headroom.proxy.savings import ItemMeasure
+
import httpx
@@ -815,6 +817,7 @@ class StreamingMixin:
# See ``conversation_savings``.
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
+ savings_measure: ItemMeasure | None = None,
) -> None:
from headroom.proxy.outcome import RequestOutcome
@@ -1040,6 +1043,7 @@ class StreamingMixin:
waste_signals=waste_signals,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ savings_measure=savings_measure,
)
await self._record_request_outcome(outcome)
@@ -1071,6 +1075,7 @@ class StreamingMixin:
session_key: str | None = None,
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
+ savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
) -> Response | StreamingResponse:
@@ -1118,6 +1123,7 @@ class StreamingMixin:
session_key=session_key,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ savings_measure=savings_measure,
server_memory_tool_names=server_memory_tool_names,
client_beta=client_beta,
)
@@ -1152,6 +1158,7 @@ class StreamingMixin:
session_key: str,
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
+ savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
) -> Response | StreamingResponse:
@@ -1586,6 +1593,7 @@ class StreamingMixin:
status_code=upstream_response.status_code,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ savings_measure=savings_measure,
)
self._cleanup_mid_turn_stream(session_key)
return Response(
@@ -1981,6 +1989,7 @@ class StreamingMixin:
),
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ savings_measure=savings_measure,
)
if supports_mid_turn_coalescing(client) and pending_messages:
pending_event = json.dumps(
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 3503d710f..837544683 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -33,6 +33,7 @@ from typing import Any
from headroom.log_safety import describe_exception
from headroom.proxy.conversation_savings import get_conversation_savings
+from headroom.proxy.savings import ItemMeasure, get_item_ledger
from headroom.proxy.tool_schema_savings_policy import (
headline_tokens_saved,
tool_schema_saved_from_tags,
@@ -204,6 +205,11 @@ class RequestOutcome:
# path, where the outcome carries a per-turn delta.
conversation_key: str | None = None
conversation_tokens_saved: int | None = None
+ # savings_measure: the request's per-item original/sent counts against the
+ # conversation's previous request (``headroom.proxy.savings``), from
+ # which the per-request savings record is priced. None on paths that
+ # do not measure items yet.
+ savings_measure: ItemMeasure | None = None
transforms_applied: tuple[str, ...] = ()
waste_signals: dict[str, int] | None = None
num_messages: int = 0
@@ -339,6 +345,7 @@ class RequestOutcome:
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
provider_input_tokens: int = 0,
+ savings_measure: ItemMeasure | None = None,
) -> RequestOutcome:
"""Construct an outcome from the locals available at streaming
finalize. Three streaming finalizers
@@ -414,6 +421,7 @@ class RequestOutcome:
tokens_saved=tokens_saved,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ savings_measure=savings_measure,
attempted_input_tokens=optimized_tokens + tokens_saved,
status_code=status_code,
cache_read_tokens=cache_read_tokens,
@@ -539,6 +547,10 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
# beacon must never add latency to, or take down, the request path.
record_outcome(outcome)
+ if outcome.savings_measure is not None:
+ # Only a request the provider served becomes the next one's baseline.
+ get_item_ledger().settle(outcome.savings_measure, succeeded=outcome.status_code < 400)
+
# A rejected turn must not feed the savings/cost/log success stats; that
# would let a request the provider never billed inflate the save-rate.
# Record it under the counter that names what happened, and stop.
diff --git a/headroom/proxy/savings.py b/headroom/proxy/savings.py
new file mode 100644
index 000000000..69a87bea2
--- /dev/null
+++ b/headroom/proxy/savings.py
@@ -0,0 +1,707 @@
+"""One per-request savings record: counterfactual cost minus actual cost.
+
+Every savings surface (/stats, the durable ledger, persistent savings, the
+dashboard, the PERF line) is meant to derive from the :class:`SavingsRecord`
+this module computes, so no surface prices savings its own way.
+
+Two rulers meet here and are only ever combined through RATIOS:
+
+* Headroom's tokenizer gives per-item counts of the request as the client sent
+ it (ORIGINAL) and as Headroom forwarded it (SENT). Both are counted with the
+ same tokenizer, so a tokenizer bias cancels in their difference.
+* The provider's usage block gives exact billed totals (uncached, cache read,
+ cache write) but no per-item detail.
+
+``k = billed / sent`` converts Headroom units into provider units, calibrated
+on this request's own content. The counterfactual cache behaviour mirrors the
+actual request's: removed tokens in items the previous request already carried
+would have been cache reads to the extent the actual request hit its cache
+(``warm``); removed tokens in new items would have been misses, written to
+cache in the proportion the actual request wrote (``w``). Old items whose sent
+copy Headroom changed since the previous request (``bust``) are misses that
+Headroom caused, charged back against the saving.
+
+Output tokens are identical in the counterfactual by construction; output
+shaping is measured separately against its holdout and stays out of this
+number.
+
+``headroom.pricing`` is imported inside functions, as elsewhere on the proxy
+path: its package ``__init__`` pulls in litellm (~4s), which must not land on
+proxy startup.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+from collections import OrderedDict
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field, replace
+from typing import TYPE_CHECKING, Any
+
+from headroom.log_safety import describe_exception, safe_id
+
+if TYPE_CHECKING:
+ from headroom.pricing.counterfactual import CacheMix, CacheRates
+
+logger = logging.getLogger(__name__)
+
+#: OpenAI caches prompts of at least 1024 tokens, in 128-token blocks.
+OPENAI_MIN_CACHEABLE_TOKENS = 1024
+OPENAI_CACHE_BLOCK_TOKENS = 128
+#: Used when the catalog has no ``prompt_cache_min_tokens`` for a model.
+DEFAULT_MIN_CACHEABLE_TOKENS = 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ItemCounts:
+ """One request item (tools block, system, or one input/message item).
+
+ Hashes identify the item's content; counts are in Headroom tokens. Lists of
+ items are aligned by position between ORIGINAL and SENT, so an item
+ Headroom dropped entirely keeps its slot with ``sent_hash=""`` and
+ ``sent_tokens=0``. Tool-schema deferral is removal inside the tools item.
+ """
+
+ original_hash: str
+ original_tokens: int
+ sent_hash: str
+ sent_tokens: int
+
+
+@dataclass(frozen=True)
+class ItemMeasure:
+ """The Headroom-unit quantities the savings formula needs."""
+
+ #: ``S``: tokens of the whole sent body.
+ sent_tokens: int
+ #: ``R_old``: removed tokens (signed: an item Headroom grew counts against
+ #: it) in items within the longest prefix of the ORIGINAL body identical to
+ #: the previous ORIGINAL body.
+ removed_old: int
+ #: ``R_new``: removed tokens in every other item, signed likewise.
+ removed_new: int
+ #: ``P_S``: tokens of the longest prefix of the SENT body identical to the
+ #: previous SENT body.
+ sent_shared_prefix: int
+ #: Sent tokens of old items that fell outside the sent shared prefix: cache
+ #: misses caused only by Headroom changing what it had sent before.
+ bust: int
+ #: False when the old/new split is approximate (the totals stay exact):
+ #: Headroom changed the item count and the original/sent pairing had to be
+ #: approximated, or the baseline was lost (``baseline_lost``).
+ aligned: bool = True
+ #: True when the ledger lost what this conversation last sent (its
+ #: baseline or an in-flight request was evicted under the budget, or its
+ #: baseline may be truncated), so ``removed_new`` may hold old items. Such
+ #: a measure is also ``aligned=False``. Its record prices the new removal
+ #: at the cache-read rate, the lowest any input token costs, so the lost
+ #: knowledge can only understate the saving. A conversation's first
+ #: request is not lost: nothing was sent before, so everything is new.
+ baseline_lost: bool = False
+ #: Tokens Headroom added to new items (the negative part of
+ #: ``removed_new``), so the read-rate floor covers only what it removed.
+ grown_new: int = 0
+ #: Which measured request this is, for :meth:`ItemLedger.settle`.
+ ticket: int = field(default=0, compare=False)
+
+ @property
+ def removed(self) -> int:
+ return self.removed_old + self.removed_new
+
+
+@dataclass(frozen=True)
+class SavingsRecord:
+ """What one request saved, in dollars and in both token rulers."""
+
+ saved_usd: float
+ actual_input_cost_usd: float
+ counterfactual_input_cost_usd: float
+ #: Removed tokens in Headroom units (``R``).
+ removed_tokens: int
+ #: The same removal in provider units (``k * R``).
+ saved_tokens: float
+ k: float
+ warm: float
+ write_share: float
+ #: True when usage was missing (aborted stream) or summed over server-side
+ #: iterations, so the cache split could not be read from this request.
+ estimated: bool = False
+ #: False when the catalog has no price for the model.
+ priced: bool = True
+
+
+def _common_prefix_len(a: Sequence[str], b: Sequence[str]) -> int:
+ n = 0
+ for x, y in zip(a, b):
+ if x != y:
+ break
+ n += 1
+ return n
+
+
+def measure_items(
+ items: Sequence[ItemCounts],
+ previous: Sequence[ItemCounts] | None,
+) -> ItemMeasure:
+ """Split one request's removal into old/new items against the previous request.
+
+ ``previous`` is the same conversation's previous request, or ``None`` on its
+ first request, where everything is new and nothing is shared.
+ """
+ n_old = _common_prefix_len(
+ [i.original_hash for i in items], [i.original_hash for i in previous or ()]
+ )
+ n_sent_shared = _common_prefix_len(
+ [i.sent_hash for i in items], [i.sent_hash for i in previous or ()]
+ )
+ removed = [i.original_tokens - i.sent_tokens for i in items]
+ sent = [max(0, i.sent_tokens) for i in items]
+ return ItemMeasure(
+ sent_tokens=sum(sent),
+ removed_old=sum(removed[:n_old]),
+ removed_new=sum(removed[n_old:]),
+ sent_shared_prefix=sum(sent[:n_sent_shared]),
+ bust=sum(sent[n_sent_shared:n_old]),
+ grown_new=-sum(r for r in removed[n_old:] if r < 0),
+ )
+
+
+def _dump(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
+
+
+#: One request item as (identity text, billable text): the identity is hashed
+#: to tell items apart, the billable text is what gets tokenized.
+Item = tuple[str, str]
+
+
+def _item(value: Any) -> Item:
+ identity = _dump(value)
+ if (
+ isinstance(value, dict)
+ and value.get("type") == "reasoning"
+ and "encrypted_content" in value
+ ):
+ # The provider bills the reasoning it decrypts, not the ciphertext,
+ # which would otherwise be most of a long Codex transcript's count.
+ billable = _dump({k: v for k, v in value.items() if k != "encrypted_content"})
+ return identity, billable
+ return identity, identity
+
+
+def responses_items(body: dict[str, Any]) -> list[Item]:
+ """A ``/v1/responses`` body as items in prompt order.
+
+ Tools, then instructions, then each ``input`` item: the order the provider
+ caches the prompt in, which is what the shared-prefix comparison needs.
+ Codex's ``additional_tools`` input items stay where the client put them.
+ """
+ items = [_item(body.get("tools") or []), _item(body.get("instructions") or "")]
+ # OpenAI-compatible upstreams accept ``messages`` in place of ``input``.
+ raw = body.get("input") if body.get("input") is not None else body.get("messages")
+ if isinstance(raw, list):
+ items.extend(_item(item) for item in raw)
+ elif raw:
+ items.append(_item(raw))
+ return items
+
+
+def _digest(text: str) -> str:
+ return hashlib.blake2b(text.encode("utf-8", errors="replace"), digest_size=8).hexdigest()
+
+
+#: Items kept across all conversations' baselines and requests awaiting their
+#: outcome, combined. A long Codex session carries 1000+ items; bounding items,
+#: not conversations, bounds memory: up to twice this while one conversation's
+#: baseline and its in-flight request exceed it (see :class:`ItemLedger`).
+MAX_STORED_ITEMS = 100_000
+
+#: Conversations remembered as having lost their baseline, a memory bound
+#: (one short key each). Past it the ledger can no longer tell a first request
+#: from a lost one and treats every request without a baseline as lost: priced
+#: at the cache-read floor, which can understate a saving but never overstate it.
+MAX_LOST_KEYS = 100_000
+
+
+def _align(
+ original: list[tuple[str, int]],
+ sent: list[tuple[str, int]],
+ previous: Sequence[ItemCounts] | None,
+) -> tuple[list[ItemCounts], bool]:
+ """Pair original and sent items, and say whether the pairing is exact.
+
+ Same item count: by position (compression rewrites items in place). A
+ different count (a hook folded or dropped items, or Headroom inserted
+ some): identical items at either end pair up; the unmatched originals in
+ between are removed, and the unmatched sent items in between are folded
+ into the first of their slots. The totals stay exact either way; only
+ which items count as old is approximate, hence ``aligned=False``. An item
+ compressed in place still pairs at the head when the previous request
+ paired the same original with the same sent copy at that position.
+
+ Equal counts pair by position even if a fold and an insert cancel out.
+ """
+ if len(original) == len(sent):
+ return [ItemCounts(oh, ot, sh, st) for (oh, ot), (sh, st) in zip(original, sent)], True
+ limit = min(len(original), len(sent))
+ prev = previous or ()
+
+ def same_as_before(i: int) -> bool:
+ return (
+ i < len(prev)
+ and original[i][0] == prev[i].original_hash
+ and sent[i][0] == prev[i].sent_hash
+ )
+
+ head = 0
+ while head < limit and (original[head][0] == sent[head][0] or same_as_before(head)):
+ head += 1
+ tail = 0
+ while (
+ tail < limit - head
+ and original[len(original) - 1 - tail][0] == sent[len(sent) - 1 - tail][0]
+ ):
+ tail += 1
+ mid_original = original[head : len(original) - tail]
+ mid_sent = sent[head : len(sent) - tail]
+ if len(mid_sent) == 1:
+ folded_hash = mid_sent[0][0]
+ else:
+ folded_hash = _digest("\x00".join(h for h, _ in mid_sent)) if mid_sent else ""
+ folded_tokens = sum(t for _, t in mid_sent)
+ middle = [ItemCounts(oh, ot, "", 0) for oh, ot in mid_original] or [ItemCounts("", 0, "", 0)]
+ middle[0] = ItemCounts(
+ middle[0].original_hash, middle[0].original_tokens, folded_hash, folded_tokens
+ )
+ pairs = [ItemCounts(oh, ot, sh, st) for (oh, ot), (sh, st) in zip(original[:head], sent[:head])]
+ pairs += middle
+ pairs += [ItemCounts(h, t, h, t) for h, t in original[len(original) - tail :]]
+ return pairs, False
+
+
+class ItemLedger:
+ """The previous request of each conversation, for the old/new item split.
+
+ One instance per process. A request is measured against the conversation's
+ last SUCCESSFUL request and becomes the baseline only once its own outcome
+ succeeds (:meth:`settle`): a failed call never reached the provider's cache.
+ Each measure carries a ticket, issued in measure order, so a slow earlier
+ turn settling late cannot replace a newer turn's baseline. Items re-sent
+ from the baseline reuse its token counts, so a long transcript is
+ tokenized once rather than every turn.
+
+ Baselines and requests awaiting their outcome share one budget,
+ :data:`MAX_STORED_ITEMS`. Past it, whichever is least recent goes first,
+ by ticket: a pending request by its own ticket (its slot is reserved when
+ the ticket is issued, so a slow measure keeps its place), a baseline by
+ the last ticket that used it. Settling counts as a use. A request being
+ admitted never evicts its own conversation's baseline: that baseline is
+ replaced only when a newer request for the conversation succeeds. When
+ only that baseline and one pending request are left, both are kept, even
+ past the budget, so a conversation larger than half the budget still
+ advances; memory then peaks at twice :data:`MAX_STORED_ITEMS` items.
+
+ A request is kept only up to its first :data:`MAX_STORED_ITEMS` items: the
+ comparison is a prefix, so the next request still gets the shared-prefix
+ credit for them.
+
+ Whatever is evicted, its conversation is remembered as lost, so its next
+ request is marked ``baseline_lost`` (and ``aligned=False``): its record
+ prices the new removal at the cache-read floor instead of as new, at the
+ miss rate. So is a request measured against a baseline of exactly
+ :data:`MAX_STORED_ITEMS` items, which may be truncated. A conversation's
+ first request is new, not lost. The memory of losses is bounded by
+ :data:`MAX_LOST_KEYS`; past it, every request without a baseline is
+ treated as lost, and a forgotten loss's stale baseline is dropped with it.
+ """
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ # key -> (settled ticket, items, last ticket that used it), oldest use first.
+ self._previous: OrderedDict[str, tuple[int, list[ItemCounts], int]] = OrderedDict()
+ # ticket -> (key, items), in ticket order.
+ self._pending: OrderedDict[int, tuple[str, list[ItemCounts]]] = OrderedDict()
+ # key -> newest ticket whose items were evicted, oldest loss first.
+ self._lost: OrderedDict[str, int] = OrderedDict()
+ # True once a loss was forgotten: a missing baseline may then be a lost one.
+ self._lost_overflowed = False
+ self._stored = 0
+ self._pending_items = 0
+ self._seq = 0
+
+ def measure(
+ self,
+ key: str,
+ original: Sequence[Item],
+ sent: Sequence[Item],
+ count: Callable[[str], int],
+ ) -> ItemMeasure:
+ """Measure one request against ``key``'s baseline; settle it later."""
+ with self._lock:
+ self._seq += 1
+ seq = self._seq
+ entry = self._previous.get(key)
+ if entry is not None:
+ self._previous[key] = (entry[0], entry[1], seq)
+ self._previous.move_to_end(key)
+ lost = key in self._lost or (entry is None and self._lost_overflowed)
+ self._pending[seq] = (key, [])
+ previous = entry[1] if entry else None
+ known: dict[str, int] = {}
+ for item in previous or ():
+ known[item.original_hash] = item.original_tokens
+ known[item.sent_hash] = item.sent_tokens
+
+ def counted(items: Sequence[Item]) -> list[tuple[str, int]]:
+ out = []
+ for identity, billable in items:
+ digest = _digest(identity)
+ tokens = known.get(digest)
+ if tokens is None:
+ tokens = known[digest] = int(count(billable))
+ out.append((digest, tokens))
+ return out
+
+ try:
+ items, aligned = _align(counted(original), counted(sent), previous)
+ except BaseException:
+ # The caller logs and never settles: release the reserved slot.
+ with self._lock:
+ self._pending.pop(seq, None)
+ raise
+ if previous is not None and len(previous) == MAX_STORED_ITEMS:
+ # Possibly a truncated baseline: items past it may be old.
+ lost = True
+ with self._lock:
+ if seq in self._pending: # not already evicted while counting
+ kept = items[:MAX_STORED_ITEMS]
+ self._pending[seq] = (key, kept)
+ self._pending_items += len(kept)
+ self._evict(key)
+ return replace(
+ measure_items(items, previous),
+ aligned=aligned and not lost,
+ baseline_lost=lost,
+ ticket=seq,
+ )
+
+ def settle(self, measure: ItemMeasure, *, succeeded: bool) -> None:
+ """Make a measured request its conversation's baseline if it succeeded."""
+ seq = measure.ticket
+ with self._lock:
+ pending = self._pending.pop(seq, None)
+ if pending is None:
+ return
+ key, items = pending
+ self._pending_items -= len(items)
+ if not succeeded:
+ return
+ current = self._previous.get(key)
+ if current is not None and current[0] > seq:
+ return
+ # Moving items from pending to stored never grows the total.
+ self._stored -= len(current[1]) if current else 0
+ self._previous[key] = (seq, items, self._seq)
+ self._previous.move_to_end(key)
+ self._stored += len(items)
+ if self._lost.get(key, seq) < seq:
+ # Newer than everything evicted: the baseline is whole again.
+ del self._lost[key]
+
+ def _evict(self, admitting: str) -> None:
+ """Forget the least recent entries until the combined budget holds.
+
+ Called with the lock held, right after admitting a request for
+ ``admitting``, whose baseline is never a candidate. Each pass removes
+ one entry until only that baseline and one pending request are left:
+ each holds at most :data:`MAX_STORED_ITEMS` items, so they stay.
+ """
+ while self._stored + self._pending_items > MAX_STORED_ITEMS:
+ oldest_ticket = next(iter(self._pending), None)
+ oldest_key = next((k for k in self._previous if k != admitting), None)
+ if oldest_key is None and len(self._pending) <= 1:
+ return
+ # On a tie the baseline goes: the request that used it is newer.
+ if oldest_key is not None and (
+ oldest_ticket is None or self._previous[oldest_key][2] <= oldest_ticket
+ ):
+ settled, evicted, _ = self._previous.pop(oldest_key)
+ self._stored -= len(evicted)
+ self._mark_lost(oldest_key, settled)
+ else:
+ ticket, (key, dropped) = self._pending.popitem(last=False)
+ self._pending_items -= len(dropped)
+ self._mark_lost(key, ticket)
+
+ def _mark_lost(self, key: str, ticket: int) -> None:
+ """Remember that ``key``'s items up to ``ticket`` were evicted (lock held)."""
+ self._lost[key] = max(ticket, self._lost.get(key, 0))
+ self._lost.move_to_end(key)
+ while len(self._lost) > MAX_LOST_KEYS:
+ forgotten, _ = self._lost.popitem(last=False)
+ if not self._lost_overflowed:
+ self._lost_overflowed = True
+ logger.info(
+ "savings: lost-baseline memory full (MAX_LOST_KEYS=%d); requests "
+ "without a baseline now price at the cache-read floor",
+ MAX_LOST_KEYS,
+ )
+ # Unmarked, a stale baseline would price old items as new: drop it,
+ # so the conversation counts as lost under the overflow rule.
+ stale = self._previous.pop(forgotten, None)
+ if stale is not None:
+ self._stored -= len(stale[1])
+
+
+_ledger = ItemLedger()
+
+
+def get_item_ledger() -> ItemLedger:
+ """Process-wide ledger: one conversation spans many requests and handlers."""
+ return _ledger
+
+
+def prefix_key(conversation: str | None, model: str, original: Sequence[Item]) -> str:
+ """Which previous request a request is compared against.
+
+ The conversation's explicit identity (``savings_conversation_key``) plus
+ the tools and instructions: a side request in the same session (compaction,
+ review) carries a different prefix and must not replace the conversation's
+ previous request. Without an identity, the model plus the first input item
+ stand in. Requests that collide on the key overwrite each other's previous
+ request, shortening the shared prefix: more of the removal is then priced
+ as new, at the miss rate, which overstates the saving.
+ """
+ head = "\x00".join(identity for identity, _ in original[:2])
+ if conversation:
+ # The model too: the provider's cache is per model, and so are counts.
+ return _digest("\x00".join([conversation, model, head]))
+ return _digest("\x00".join([model, *(identity for identity, _ in original[:3])]))
+
+
+def _input_cost(
+ rates: CacheRates, *, uncached: float, read: float, write_5m: float, write_1h: float
+) -> float:
+ return (
+ uncached * rates.uncached
+ + read * rates.read
+ + write_5m * rates.write_5m
+ + write_1h * rates.write_1h
+ )
+
+
+def _min_cacheable_tokens(model: str, provider: str | None) -> int:
+ if provider == "openai":
+ return OPENAI_MIN_CACHEABLE_TOKENS
+ from headroom.pricing.counterfactual import catalog_min_cacheable_tokens
+
+ return catalog_min_cacheable_tokens(model) or DEFAULT_MIN_CACHEABLE_TOKENS
+
+
+def _unpriced(measure: ItemMeasure, *, k: float, estimated: bool) -> SavingsRecord:
+ return SavingsRecord(
+ saved_usd=0.0,
+ actual_input_cost_usd=0.0,
+ counterfactual_input_cost_usd=0.0,
+ removed_tokens=measure.removed,
+ saved_tokens=k * measure.removed,
+ k=k,
+ warm=0.0,
+ write_share=0.0,
+ estimated=estimated,
+ priced=False,
+ )
+
+
+def compute_savings_record(
+ measure: ItemMeasure,
+ usage: CacheMix | None,
+ *,
+ model: str,
+ provider: str | None = None,
+ estimated: bool = False,
+) -> SavingsRecord:
+ """Price one request's removal as ``counterfactual_cost - actual_cost``.
+
+ ``usage`` is the provider's billed input as a :class:`CacheMix` (normalized
+ here, so an inferred OpenAI write counts as the uncached slice it is).
+ ``None`` or an empty mix means the provider reported nothing (an aborted
+ stream); the record then prices at list with ``k = 1`` and is marked
+ ``estimated``.
+
+ For a request the provider ran as several server-side iterations, pass the
+ FIRST iteration's usage when it is available (``estimated=False``); the
+ later iterations are a cost the caller books separately. Pass the summed
+ usage with ``estimated=True`` only when per-iteration usage is not
+ available: the sum describes more prompts than the one sent body, so
+ ``k`` is not calibrated from it (``k = 1``) and only its cache ratios are
+ used. ``provider`` must be given for OpenAI's 128-token cache blocks to
+ apply.
+
+ Never raises: a savings figure must not be able to fail a request.
+ """
+ try:
+ record = _compute(measure, usage, model=model, provider=provider, estimated=estimated)
+ # An approximate item pairing leaves the totals exact but not the
+ # old/new split the price depends on.
+ return record if measure.aligned else replace(record, estimated=True)
+ except Exception as exc: # noqa: BLE001 - pricing must never fail a request
+ logger.debug(
+ "savings: record failed for model=%s: %s", safe_id(model), describe_exception(exc)
+ )
+ return _unpriced(measure, k=1.0, estimated=True)
+
+
+def _compute(
+ measure: ItemMeasure,
+ usage: CacheMix | None,
+ *,
+ model: str,
+ provider: str | None,
+ estimated: bool,
+) -> SavingsRecord:
+ from headroom.pricing.counterfactual import LONG_CONTEXT_THRESHOLD_TOKENS, resolve_rates
+
+ def rates_for(billed: float, *, for_billing: bool = False) -> CacheRates | None:
+ return resolve_rates(
+ model,
+ long_context=billed > LONG_CONTEXT_THRESHOLD_TOKENS,
+ provider=provider,
+ for_billing=for_billing,
+ )
+
+ removed = measure.removed
+ # With the baseline lost, the new items may be old ones the provider had
+ # cached. Their removal prices at the read rate, the floor of any input
+ # token: the cache ratio and minimum below describe a prefix the ledger no
+ # longer knows. Tokens Headroom added to new items stay at the miss rate.
+ floor = measure.removed_new + measure.grown_new if measure.baseline_lost else 0
+ mix = usage.normalized() if usage is not None else None
+ if mix is None or mix.billed <= 0:
+ actual_rates = rates_for(measure.sent_tokens)
+ cf_rates = rates_for(measure.sent_tokens + removed)
+ if actual_rates is None or cf_rates is None:
+ return _unpriced(measure, k=1.0, estimated=True)
+ actual = measure.sent_tokens * actual_rates.uncached
+ counterfactual = (
+ measure.sent_tokens + removed - floor
+ ) * cf_rates.uncached + floor * cf_rates.read
+ return SavingsRecord(
+ saved_usd=counterfactual - actual,
+ actual_input_cost_usd=actual,
+ counterfactual_input_cost_usd=counterfactual,
+ removed_tokens=removed,
+ saved_tokens=float(removed),
+ k=1.0,
+ warm=0.0,
+ write_share=0.0,
+ estimated=True,
+ )
+
+ billed = mix.billed
+ writes = mix.write_5m + mix.write_1h
+ k = billed / measure.sent_tokens if measure.sent_tokens > 0 and not estimated else 1.0
+ hit = mix.read / billed
+ sigma = measure.sent_shared_prefix / measure.sent_tokens if measure.sent_tokens > 0 else 0.0
+ warm = min(1.0, hit / sigma) if sigma > 0 else 0.0
+ misses = mix.uncached + writes
+ w = writes / misses if misses > 0 else 0.0
+
+ # The counterfactual must obey its own cache minimum. The catalog's
+ # minimum is used unless this request was demonstrably cached or written
+ # below it; then the catalog is wrong and no minimum is applied, the
+ # direction that cannot inflate the saving.
+ min_cacheable = _min_cacheable_tokens(model, provider)
+ proven = mix.read if mix.read > 0 else (billed if writes > 0 else min_cacheable)
+ threshold = min_cacheable if proven >= min_cacheable else 0
+ # Removed or busted old tokens could only have been reads if the
+ # counterfactual's old prefix was itself cacheable.
+ raw_warm = warm
+ if k * (measure.sent_shared_prefix + measure.removed_old) < threshold:
+ warm = 0.0
+
+ # The counterfactual's buckets are built whole, not as deltas clamped onto
+ # the actual ones: a counterfactual smaller than the request (an old item
+ # Headroom grew) must not inherit reads or writes it was too small to make.
+ # Its reads are the actual reads plus the removed/busted old tokens that
+ # would have hit, minus the reads a grown old item contributed (those go
+ # whatever the prefix size), never more than its whole billed input, and
+ # none below its cache minimum. ``cf_billed`` is ``k * original`` when k is
+ # calibrated; the clamp matters only for summed (estimated) usage.
+ cf_billed = max(0.0, billed + k * removed)
+ gained = k * (max(0, measure.removed_old) + measure.bust) * warm
+ lost = k * min(0, measure.removed_old) * raw_warm
+ cf_read = min(max(0.0, mix.read + gained + lost), cf_billed)
+ if cf_read < threshold:
+ # Never fires when nothing changed: the actual reads are 0 or proven
+ # at least the threshold.
+ cf_read = 0.0
+ if provider == "openai" and cf_read != mix.read:
+ # Hits come in whole 128-token blocks: round down, but a counterfactual
+ # that adds reads keeps at least the actual ones. The ``!=`` keeps an
+ # unchanged request exactly at the actual reads.
+ blocks = (cf_read // OPENAI_CACHE_BLOCK_TOKENS) * OPENAI_CACHE_BLOCK_TOKENS
+ cf_read = max(blocks, mix.read) if cf_read > mix.read else blocks
+ if floor > 0:
+ cf_read = min(cf_read + k * floor, cf_billed)
+ # Everything else missed, written in the proportion the request wrote,
+ # unless the counterfactual is smaller than anything shown cacheable: a
+ # write it could not make would inflate the saving.
+ cf_miss = cf_billed - cf_read
+ cf_write = cf_miss * w if cf_billed >= min(min_cacheable, proven) else 0.0
+ w5_share = mix.write_5m / writes if writes > 0 else 1.0
+ cf_buckets = {
+ "uncached": cf_miss - cf_write,
+ "read": cf_read,
+ "write_5m": cf_write * w5_share,
+ "write_1h": cf_write * (1.0 - w5_share),
+ }
+ actual_buckets = {
+ "uncached": float(mix.uncached),
+ "read": float(mix.read),
+ "write_5m": float(mix.write_5m),
+ "write_1h": float(mix.write_1h),
+ }
+
+ # Tokens the request was billed for price at billing rates on both sides
+ # (the counterfactual's at its own context tier); only the tokens a
+ # counterfactual bucket holds beyond the actual one price at the
+ # savings-estimate rates. Every priced quantity is nonnegative.
+ actual_rates = rates_for(billed, for_billing=True)
+ cf_billing_rates = rates_for(cf_billed, for_billing=True)
+ cf_rates = rates_for(cf_billed)
+ if actual_rates is None or cf_billing_rates is None or cf_rates is None:
+ return _unpriced(measure, k=k, estimated=estimated)
+
+ actual = _input_cost(actual_rates, **actual_buckets)
+ counterfactual = _input_cost(
+ cf_billing_rates, **{b: min(n, actual_buckets[b]) for b, n in cf_buckets.items()}
+ ) + _input_cost(cf_rates, **{b: max(0.0, n - actual_buckets[b]) for b, n in cf_buckets.items()})
+ return SavingsRecord(
+ saved_usd=counterfactual - actual,
+ actual_input_cost_usd=actual,
+ counterfactual_input_cost_usd=counterfactual,
+ removed_tokens=removed,
+ saved_tokens=k * removed,
+ k=k,
+ warm=warm,
+ write_share=w,
+ estimated=estimated,
+ )
+
+
+__all__ = [
+ "Item",
+ "ItemCounts",
+ "ItemLedger",
+ "ItemMeasure",
+ "SavingsRecord",
+ "compute_savings_record",
+ "get_item_ledger",
+ "measure_items",
+ "prefix_key",
+ "responses_items",
+]
diff --git a/tests/test_savings_items.py b/tests/test_savings_items.py
new file mode 100644
index 000000000..f4166d421
--- /dev/null
+++ b/tests/test_savings_items.py
@@ -0,0 +1,376 @@
+"""Per-item original/sent measurement feeding the savings record."""
+
+from __future__ import annotations
+
+import json
+import threading
+from typing import Any
+
+import httpx
+import pytest
+import respx
+from fastapi.testclient import TestClient
+
+from headroom.proxy import savings
+from headroom.proxy.loopback_guard import require_loopback
+from headroom.proxy.savings import ItemLedger, ItemMeasure, prefix_key, responses_items
+from headroom.proxy.server import ProxyConfig, create_app
+
+
+def _words(text: str) -> int:
+ return len(text.split(","))
+
+
+def _items(*texts: str) -> list[tuple[str, str]]:
+ return [(t, t) for t in texts]
+
+
+def _turn(ledger, original, sent, count=_words, *, succeeded=True):
+ """Measure one request and settle it, as the outcome funnel does."""
+ m = ledger.measure("conv", original, sent, count)
+ ledger.settle(m, succeeded=succeeded)
+ return m
+
+
+def test_a_conversation_is_measured_against_its_previous_request():
+ ledger = ItemLedger()
+ calls: list[str] = []
+
+ def count(text: str) -> int:
+ calls.append(text)
+ return _words(text)
+
+ first = ['["t1","t2"]', '"sys"', '{"a":1,"b":2,"c":3}']
+ sent1 = ['["t1"]', '"sys"', '{"a":1,"b":2,"c":3}']
+ second = [*first, '{"d":4,"e":5}']
+ sent2 = [*sent1, '{"d":4}']
+
+ assert _turn(ledger, _items(*first), _items(*sent1), count) == ItemMeasure(
+ sent_tokens=5, removed_old=0, removed_new=1, sent_shared_prefix=0, bust=0
+ )
+ assert _turn(ledger, _items(*second), _items(*sent2), count) == ItemMeasure(
+ sent_tokens=6, removed_old=1, removed_new=1, sent_shared_prefix=5, bust=0
+ )
+ # Re-sent items are tokenized once, not once per turn.
+ assert len(calls) == len(set(calls))
+
+
+def test_items_follow_the_providers_prompt_order():
+ body = {"tools": [{"name": "t"}], "instructions": "be brief", "input": [{"x": 1}, {"y": 2}]}
+ assert responses_items(body) == _items('[{"name":"t"}]', '"be brief"', '{"x":1}', '{"y":2}')
+ assert responses_items({"input": "hi"}) == _items("[]", '""', '"hi"')
+
+
+def test_reasoning_ciphertext_is_identity_not_billed_text():
+ """Codex replays encrypted reasoning every turn; counting the ciphertext
+ made Headroom's count drift far above the provider's bill."""
+ item = {"type": "reasoning", "summary": [], "encrypted_content": "gAAAA" * 50}
+ [(identity, billable)] = responses_items({"input": [item]})[2:]
+ assert "gAAAA" in identity
+ assert billable == '{"type":"reasoning","summary":[]}'
+
+
+def test_a_folded_transcript_keeps_the_old_items_old():
+ """A hook folding two items into one shifts every later position; pairing
+ by position then priced untouched new items as removed."""
+ ledger = ItemLedger()
+ t, i = "[]", '"sys"'
+ a, b, c, d = ("," * 9 + x for x in "abcd") # 10 tokens each under _words
+ _turn(ledger, _items(t, i, a, b, c), _items(t, i, a, b, c), _words)
+ m = _turn(ledger, _items(t, i, a, b, c, d), _items(t, i, "f", c, d), _words)
+ assert (m.removed_old, m.removed_new) == (19, 0)
+ # The fold moved c: its sent copy is outside the shared prefix, a bust.
+ assert m.bust == 11 and m.aligned is False
+
+
+def test_a_fold_after_an_item_compressed_in_place_keeps_the_prefix():
+ """The common Codex case: an early tool output compressed every turn, then
+ a hook folds the new items. The compressed item must still pair at the head,
+ or every later item is priced as removed and as a bust."""
+ ledger = ItemLedger()
+ t, i = "[]", '"sys"'
+ big, small = "," * 99 + "x", "x" # 100 and 1 tokens under _words
+ n1, n2 = "," * 9 + "n", "," * 9 + "m"
+ _turn(ledger, _items(t, i, big), _items(t, i, small), _words)
+ m = _turn(ledger, _items(t, i, big, n1, n2), _items(t, i, small, "f"), _words)
+ assert (m.sent_shared_prefix, m.bust) == (3, 0)
+ assert (m.removed_old, m.removed_new) == (99, 19)
+ assert m.aligned is False
+
+
+def test_an_approximate_pairing_marks_the_record_estimated():
+ m = ItemMeasure(
+ sent_tokens=10, removed_old=0, removed_new=5, sent_shared_prefix=0, bust=0, aligned=False
+ )
+ assert savings.compute_savings_record(m, None, model="unpriced-model").estimated
+
+
+def test_a_failed_request_does_not_become_the_baseline():
+ ledger = ItemLedger()
+ first = _items("[]", '"sys"', '"a"')
+ _turn(ledger, first, first)
+ _turn(ledger, _items("[]", '"other"', '"b"'), _items("[]", '"other"', '"b"'), succeeded=False)
+ assert _turn(ledger, first, first).sent_shared_prefix == 3
+
+
+def test_a_late_settling_older_turn_keeps_the_newer_baseline():
+ ledger = ItemLedger()
+ old, new = _items("[]", '"a"'), _items("[]", '"a"', '"b"')
+ slow = ledger.measure("conv", old, old, _words)
+ fast = ledger.measure("conv", new, new, _words)
+ ledger.settle(fast, succeeded=True)
+ ledger.settle(slow, succeeded=True)
+ assert _turn(ledger, new, new).sent_shared_prefix == 3
+
+
+def test_requests_awaiting_their_outcome_are_bounded_by_items(monkeypatch):
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ abandoned = ledger.measure("a", _items("[]", '"x"', '"y"'), _items("[]", '"x"', '"y"'), _words)
+ kept = ledger.measure("b", _items("[]", '"z"'), _items("[]", '"z"'), _words)
+ # The oldest pending request was forgotten; settling it is a no-op.
+ ledger.settle(abandoned, succeeded=True)
+ ledger.settle(kept, succeeded=True)
+ # Each probe is itself a pending request under the budget: kept first.
+ assert (
+ ledger.measure("b", _items("[]", '"z"'), _items("[]", '"z"'), _words).sent_shared_prefix
+ == 2
+ )
+ again = _items("[]", '"x"', '"y"')
+ assert ledger.measure("a", again, again, _words).sent_shared_prefix == 0
+
+
+def test_a_slow_older_measure_never_evicts_a_newer_one(monkeypatch):
+ """JerrettDavis's #4240 interleaving: ticket 1 stalls inside token
+ counting while ticket 2 measures four items. Eviction must drop the older
+ ticket, not the newer one that finished first."""
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ counting, release = threading.Event(), threading.Event()
+
+ def stalled(text: str) -> int:
+ counting.set()
+ assert release.wait(timeout=10)
+ return _words(text)
+
+ older: list[ItemMeasure] = []
+ slow = _items("[]", '"x"')
+ thread = threading.Thread(target=lambda: older.append(ledger.measure("a", slow, slow, stalled)))
+ thread.start()
+ assert counting.wait(timeout=10)
+ new = _items("[]", '"b1"', '"b2"', '"b3"')
+ newer = ledger.measure("b", new, new, _words)
+ release.set()
+ thread.join(timeout=10)
+
+ ledger.settle(newer, succeeded=True)
+ ledger.settle(older[0], succeeded=True)
+ assert ledger.measure("b", new, new, _words).sent_shared_prefix == 4
+
+
+def test_settled_and_pending_items_share_one_budget(monkeypatch):
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ idle = _items("[]", '"a1"', '"a2"', '"a3"')
+ _turn(ledger, idle, idle)
+ active = _items("[]", '"b1"', '"b2"', '"b3"')
+ in_flight = ledger.measure("b", active, active, _words)
+ # Admitting the in-flight request evicted the idle baseline at once.
+ assert ledger._stored + ledger._pending_items == 4
+ ledger.settle(in_flight, succeeded=True)
+ assert ledger.measure("b", active, active, _words).sent_shared_prefix == 4
+ assert ledger.measure("conv", idle, idle, _words).sent_shared_prefix == 0
+
+
+def test_a_request_larger_than_the_budget_is_kept_up_to_the_budget(monkeypatch):
+ """Keeping nothing would leave the next request with no baseline. The
+ first items are kept instead; a request measured against them may have
+ old items past the cut, so its baseline counts as lost. Each turn still
+ replaces the baseline, however large the conversation."""
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ big = _items("[]", '"s"', '"u1"', '"u2"', '"u3"')
+ m = _turn(ledger, big, big)
+ assert (m.aligned, m.baseline_lost) == (True, False)
+ longer = [*big, *_items('"u4"')]
+ m = _turn(ledger, longer, longer)
+ assert (m.sent_shared_prefix, m.aligned, m.baseline_lost) == (4, False, True)
+ assert not ledger._lost # the cut-off rule fired, nothing was evicted
+ assert ledger._stored + ledger._pending_items <= 4
+
+
+def test_a_failed_turn_keeps_its_conversations_baseline(monkeypatch):
+ """Devin's #4240 example: admitting a four-item turn under a four-item
+ budget evicted the conversation's own baseline; the turn then got a 429,
+ and the next identical request saw no shared prefix at all."""
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ turn = _items("[]", '"s"', '"u1"', '"u2"')
+ _turn(ledger, turn, turn)
+ _turn(ledger, turn, turn, succeeded=False)
+ assert _turn(ledger, turn, turn).sent_shared_prefix == 4
+
+
+def test_overlapping_turns_keep_the_baseline_until_one_succeeds(monkeypatch):
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 8)
+ ledger = ItemLedger()
+ base = _items("[]", '"s"', '"u1"')
+ _turn(ledger, base, base)
+ first = [*base, *_items('"u2"')]
+ second = [*first, *_items('"u3"', '"u4"')]
+ m1 = ledger.measure("conv", first, first, _words)
+ # Three baseline items plus ten pending: the older turn goes, not the baseline.
+ m2 = ledger.measure("conv", second, second, _words)
+ ledger.settle(m2, succeeded=False)
+ ledger.settle(m1, succeeded=False)
+ m3 = ledger.measure("conv", first, first, _words)
+ # Still the old baseline, but the forgotten turn may have been cached.
+ assert (m3.sent_shared_prefix, m3.baseline_lost, m3.aligned) == (3, True, False)
+ ledger.settle(m3, succeeded=True)
+ m4 = _turn(ledger, second, second)
+ assert (m4.sent_shared_prefix, m4.baseline_lost, m4.aligned) == (4, False, True)
+
+
+def _probe(ledger, key, items):
+ m = ledger.measure(key, items, items, _words)
+ ledger.settle(m, succeeded=True)
+ return m
+
+
+def test_a_lost_baseline_is_flagged_and_a_new_conversation_is_not(monkeypatch):
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ ledger = ItemLedger()
+ a, b, c = (_items("[]", f'"{n}1"', f'"{n}2"') for n in "abc")
+ assert not _probe(ledger, "a", a).baseline_lost
+ # Admitting b evicts a's baseline: another conversation, the least recent.
+ assert not _probe(ledger, "b", b).baseline_lost
+ again = _probe(ledger, "a", a)
+ assert (again.sent_shared_prefix, again.baseline_lost, again.aligned) == (0, True, False)
+ assert not _probe(ledger, "c", c).baseline_lost
+
+
+def test_past_the_loss_bound_every_missing_baseline_counts_as_lost(monkeypatch):
+ """Forgetting a loss must not let its conversation pass for a new one."""
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ monkeypatch.setattr(savings, "MAX_LOST_KEYS", 1)
+ ledger = ItemLedger()
+ a, b, c, d = (_items("[]", f'"{n}1"', f'"{n}2"') for n in "abcd")
+ _probe(ledger, "a", a)
+ _probe(ledger, "b", b) # loses a
+ _probe(ledger, "c", c) # loses b; a's loss is forgotten
+ assert _probe(ledger, "a", a).baseline_lost
+ assert _probe(ledger, "d", d).baseline_lost
+
+
+def test_a_forgotten_loss_drops_the_stale_baseline_it_leaves(monkeypatch):
+ """a's newer request was evicted while its baseline survived. Once that
+ loss is forgotten, the stale baseline must not pass for a whole one, or
+ the evicted request's cached items would price as new."""
+ monkeypatch.setattr(savings, "MAX_STORED_ITEMS", 4)
+ monkeypatch.setattr(savings, "MAX_LOST_KEYS", 1)
+ ledger = ItemLedger()
+ one, two = _items('"x"'), _items("[]", '"y"')
+ _probe(ledger, "x", one)
+ _probe(ledger, "a", one)
+ a1 = ledger.measure("a", one, one, _words)
+ ledger.settle(ledger.measure("x", one, one, _words), succeeded=False) # x used after a1
+ a2 = ledger.measure("a", one, one, _words)
+ ledger.measure("b", one, one, _words) # evicts a1: a is lost, its baseline stays
+ ledger.settle(a1, succeeded=True)
+ ledger.settle(a2, succeeded=False)
+ assert "a" in ledger._lost
+ ledger.measure("c", two, two, _words) # evicts x's baseline: a's loss is forgotten
+ m = ledger.measure("a", one, one, _words)
+ assert (m.baseline_lost, m.aligned) == (True, False)
+
+
+def test_a_measure_that_fails_releases_its_slot():
+ """The caller logs a failed measure and never settles it."""
+ ledger = ItemLedger()
+
+ def broken(text: str) -> int:
+ raise ValueError("tokenizer failed")
+
+ for _ in range(3):
+ with pytest.raises(ValueError):
+ ledger.measure("conv", _items("[]"), _items("[]"), broken)
+ assert not ledger._pending
+
+
+def test_messages_stand_in_for_input():
+ assert responses_items({"messages": [{"role": "user", "content": "hi"}]})[2:] == _items(
+ '{"role":"user","content":"hi"}'
+ )
+
+
+def test_side_requests_do_not_replace_the_conversations_previous_request():
+ main = _items("[]", '"agent"', '"hi"')
+ side = _items("[]", '"summarize"', '"hi"')
+ assert prefix_key("conv-1", "m", main) != prefix_key("conv-1", "m", side)
+ assert prefix_key("conv-1", "m", main) == prefix_key(
+ "conv-1", "m", [*main, ('"more"', '"more"')]
+ )
+ assert prefix_key(None, "m", main) == prefix_key(None, "m", [*main, ('"more"', '"more"')])
+ assert prefix_key(None, "m", main) != prefix_key(None, "other", main)
+
+
+def _sse(usage: dict) -> str:
+ done = {"type": "response.completed", "response": {"id": "r", "output": [], "usage": usage}}
+ return f"event: response.completed\ndata: {json.dumps(done)}\n\n"
+
+
[email protected]("stream", [False, True])
+def test_responses_outcomes_carry_the_measure(monkeypatch, stream):
+ app = create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+ )
+ app.dependency_overrides[require_loopback] = lambda: None
+ outcomes: list[Any] = []
+
+ # Capture at the funnel's first effect, so the real funnel (which settles
+ # the measure) still runs.
+ monkeypatch.setattr("headroom.telemetry.session.record_outcome", outcomes.append)
+ monkeypatch.setattr(savings, "_ledger", ItemLedger())
+ usage = {"input_tokens": 10, "output_tokens": 1}
+ respx.post("https://api.openai.com/v1/responses").mock(
+ return_value=httpx.Response(
+ 200,
+ text=_sse(usage),
+ headers={"content-type": "text/event-stream"},
+ )
+ if stream
+ else httpx.Response(200, json={"id": "r", "output": [], "usage": usage})
+ )
+ turn1 = [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "a"}]}]
+ turn2 = [
+ *turn1,
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "b"}]},
+ ]
+ with TestClient(app) as client:
+ for items in (turn1, turn2):
+ r = client.post(
+ "/v1/responses",
+ json={
+ "model": "gpt-4o",
+ "instructions": "be brief",
+ "input": items,
+ "stream": stream,
+ "client_metadata": {"session_id": "conv-test"},
+ },
+ headers={"authorization": "Bearer sk-test"},
+ )
+ assert r.status_code == 200
+ r.read()
+
+ first, second = (o.savings_measure for o in outcomes)
+ assert first.removed == 0 and first.sent_shared_prefix == 0
+ # Nothing was compressed, so the whole first request is the shared prefix.
+ assert second.sent_shared_prefix == first.sent_tokens
+ assert second.removed == 0 and second.bust == 0
diff --git a/tests/test_savings_record.py b/tests/test_savings_record.py
new file mode 100644
index 000000000..b371522a4
--- /dev/null
+++ b/tests/test_savings_record.py
@@ -0,0 +1,368 @@
+"""The per-request savings record: counterfactual cost minus actual cost."""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+
+from headroom.pricing import counterfactual
+from headroom.pricing.counterfactual import CacheMix, CacheRates
+from headroom.proxy.savings import (
+ ItemCounts,
+ ItemMeasure,
+ compute_savings_record,
+ measure_items,
+)
+
+# $3/M input, $0.30/M cache read, $3.75/M 5m write, $6/M 1h write; the
+# long-context tier doubles everything.
+IN, READ, WRITE, WRITE_1H = 3e-6, 3e-7, 3.75e-6, 6e-6
+BASE = CacheRates(read=READ, write_5m=WRITE, write_1h=WRITE_1H, uncached=IN)
+LONG = CacheRates(read=2 * READ, write_5m=2 * WRITE, write_1h=2 * WRITE_1H, uncached=2 * IN)
+
+
[email protected](autouse=True)
+def _fixed_catalog(monkeypatch):
+ """Pin the catalog so the arithmetic is tested whatever litellm ships."""
+
+ def rates(model, *, long_context=False, provider=None, for_billing=False):
+ return None if model == "unpriced" else (LONG if long_context else BASE)
+
+ monkeypatch.setattr(counterfactual, "resolve_rates", rates)
+ monkeypatch.setattr(
+ counterfactual,
+ "catalog_min_cacheable_tokens",
+ lambda model: 4096 if model == "haiku" else 1024,
+ )
+
+
+def _formula(m: ItemMeasure, u: CacheMix) -> float:
+ """The closed form the record must agree with."""
+ billed = u.read + u.write_5m + u.uncached
+ k = billed / m.sent_tokens
+ warm = min(1.0, (u.read / billed) / (m.sent_shared_prefix / m.sent_tokens))
+ w = u.write_5m / (u.uncached + u.write_5m)
+ p_miss = w * WRITE + (1 - w) * IN
+ return (
+ k * m.removed_old * (warm * READ + (1 - warm) * p_miss)
+ + k * m.removed_new * p_miss
+ - k * m.bust * warm * (p_miss - READ)
+ )
+
+
+WARM = CacheMix(uncached=2_000, read=90_000, write_5m=8_000)
+MEASURE = ItemMeasure(
+ sent_tokens=95_000, removed_old=50_000, removed_new=10_000, sent_shared_prefix=88_000, bust=0
+)
+
+
+def test_worked_example():
+ record = compute_savings_record(MEASURE, WARM, model="sonnet", provider="anthropic")
+ assert record.k == pytest.approx(1.0526, abs=1e-4)
+ assert record.warm == pytest.approx(0.97159, abs=1e-5)
+ assert record.write_share == pytest.approx(0.8)
+ assert record.saved_usd == pytest.approx(0.058618, abs=1e-6)
+ assert record.saved_usd == pytest.approx(_formula(MEASURE, WARM), abs=1e-9)
+ assert record.saved_usd == pytest.approx(
+ record.counterfactual_input_cost_usd - record.actual_input_cost_usd
+ )
+ assert record.saved_tokens == pytest.approx(100_000 / 95_000 * 60_000)
+ assert not record.estimated
+
+
+def test_a_cache_bust_headroom_caused_is_charged_against_it():
+ busted = dataclasses.replace(MEASURE, bust=5_000)
+ record = compute_savings_record(busted, WARM, model="sonnet", provider="anthropic")
+ assert record.saved_usd == pytest.approx(_formula(busted, WARM), abs=1e-9)
+ assert record.saved_usd == pytest.approx(0.058618 - 0.016875, abs=1e-6)
+
+
+def test_a_bust_never_claims_more_reads_than_the_request_missed():
+ """Item-level P_S undercounts a token-level hit, so hit > sigma clips warm
+ to 1; the bust charge must still not drive the counterfactual negative."""
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=0, removed_new=0, sent_shared_prefix=2_000, bust=8_000
+ )
+ usage = CacheMix(uncached=500, read=9_000, write_5m=500)
+ record = compute_savings_record(measure, usage, model="sonnet", provider="anthropic")
+ assert record.counterfactual_input_cost_usd >= 0
+ # At most the 1,000 missed tokens could have been reads instead.
+ assert record.saved_usd == pytest.approx(-(500 * (IN - READ) + 500 * (WRITE - READ)))
+
+
+def test_an_old_item_headroom_grew_inherits_no_reads_below_the_cache_minimum():
+ """JerrettDavis's #4236 case: a 1,000-token item sent as 10,000 tokens on
+ successive requests, prompt cached. The 1,000-token counterfactual is
+ under the 1,024-token minimum, so none of it could have been a read."""
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0
+ )
+ usage = CacheMix(read=9_000, uncached=1_000)
+ for provider in ("openai", "anthropic"):
+ record = compute_savings_record(measure, usage, model="gpt", provider=provider)
+ assert record.actual_input_cost_usd == pytest.approx(0.0057)
+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN) # $0.003
+ assert record.saved_usd == pytest.approx(-0.0027)
+
+
+def test_summed_iteration_usage_does_not_calibrate_k():
+ """Usage summed over server-side iterations describes more prompts than
+ the one sent body, so it must not scale the removal."""
+ summed = CacheMix(read=270_000, uncached=6_000, write_5m=24_000)
+ record = compute_savings_record(
+ MEASURE, summed, model="sonnet", provider="anthropic", estimated=True
+ )
+ assert record.k == 1.0
+ assert record.estimated
+ assert record.saved_tokens == 60_000
+
+
+def test_a_cold_cache_prices_every_removed_token_as_a_miss():
+ cold = CacheMix(uncached=10_000, read=0, write_5m=90_000)
+ record = compute_savings_record(MEASURE, cold, model="sonnet", provider="anthropic")
+ assert record.warm == 0.0
+ p_miss = 0.9 * WRITE + 0.1 * IN
+ assert record.saved_usd == pytest.approx(100_000 / 95_000 * 60_000 * p_miss)
+
+
+def test_one_hour_writes_price_at_their_own_rate():
+ usage = CacheMix(uncached=2_000, read=90_000, write_1h=8_000)
+ record = compute_savings_record(MEASURE, usage, model="sonnet", provider="anthropic")
+ k, warm = 100_000 / 95_000, record.warm
+ p_miss = 0.8 * WRITE_1H + 0.2 * IN
+ expected = k * 50_000 * (warm * READ + (1 - warm) * p_miss) + k * 10_000 * p_miss
+ assert record.saved_usd == pytest.approx(expected)
+
+
+def test_an_inferred_openai_write_is_the_uncached_slice_not_a_write():
+ usage = CacheMix(read=90_000, write_total=10_000, uncached=10_000, inferred_write=True)
+ record = compute_savings_record(MEASURE, usage, model="gpt", provider="openai")
+ assert record.k == pytest.approx(100_000 / 95_000)
+ assert record.write_share == 0.0
+
+
+def test_openai_reads_come_in_128_token_blocks():
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=1_000, removed_new=0, sent_shared_prefix=10_000, bust=0
+ )
+ usage = CacheMix(read=9_984, uncached=16) # 78 blocks
+ record = compute_savings_record(measure, usage, model="gpt", provider="openai")
+ # 1,000 * k removed reads would end mid-block: only whole blocks count.
+ total_read = 9_984 + 1_000 * record.k
+ reads = (total_read // 128) * 128 - 9_984
+ assert record.saved_usd == pytest.approx(reads * READ + (1_000 * record.k - reads) * IN)
+
+
+def test_a_counterfactual_prefix_below_the_models_cache_minimum_is_never_warm():
+ # A 3,000-token old prefix: cacheable on Sonnet (1,024), not on Haiku (4,096).
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=1_000, removed_new=0, sent_shared_prefix=2_000, bust=0
+ )
+ usage = CacheMix(uncached=5_000, read=5_000)
+ assert compute_savings_record(measure, usage, model="sonnet").warm > 0
+ assert compute_savings_record(measure, usage, model="haiku").warm == 0.0
+
+
+def test_the_counterfactual_is_priced_at_its_own_context_tier():
+ measure = ItemMeasure(
+ sent_tokens=150_000, removed_old=0, removed_new=100_000, sent_shared_prefix=0, bust=0
+ )
+ record = compute_savings_record(measure, CacheMix(uncached=150_000), model="sonnet")
+ assert record.actual_input_cost_usd == pytest.approx(150_000 * IN)
+ assert record.counterfactual_input_cost_usd == pytest.approx(250_000 * 2 * IN)
+
+
+def test_missing_usage_prices_at_list_per_tier_and_says_so():
+ measure = dataclasses.replace(MEASURE, sent_tokens=150_000, removed_new=50_000)
+ record = compute_savings_record(measure, None, model="sonnet")
+ assert record.estimated
+ assert record.actual_input_cost_usd == pytest.approx(150_000 * IN)
+ assert record.counterfactual_input_cost_usd == pytest.approx(250_000 * 2 * IN)
+
+
+def test_an_unpriced_model_reports_tokens_but_no_dollars():
+ record = compute_savings_record(MEASURE, WARM, model="unpriced")
+ assert not record.priced
+ assert record.saved_usd == 0.0
+ assert record.removed_tokens == 60_000
+
+
+def test_items_split_into_old_and_new_against_the_previous_request():
+ def item(name: str, original: int, sent: int, sent_name: str | None = None) -> ItemCounts:
+ return ItemCounts(name, original, sent_name or name + "'", sent)
+
+ first = [item("tools", 1_000, 200), item("sys", 500, 500, "sys"), item("u1", 300, 300, "u1")]
+ second = [
+ item("tools", 1_000, 200),
+ item("sys", 500, 500, "sys"),
+ # Headroom compressed this item differently this time: its sent copy
+ # changed, so the provider missed it although the client resent it.
+ item("u1", 300, 250, "u1-recompressed"),
+ item("t1", 4_000, 1_000),
+ # An item Headroom grew counts against the removal.
+ item("t2", 100, 160),
+ ]
+ assert measure_items(first, None) == ItemMeasure(
+ sent_tokens=1_000, removed_old=0, removed_new=800, sent_shared_prefix=0, bust=0
+ )
+ assert measure_items(second, first) == ItemMeasure(
+ sent_tokens=2_110,
+ removed_old=800 + 50,
+ removed_new=3_000 - 60,
+ sent_shared_prefix=700,
+ bust=250,
+ grown_new=60,
+ )
+
+
+def test_billed_buckets_price_at_billing_rates(monkeypatch):
+ """A catalog row without a cache-read rate bills reads via litellm's
+ calculator (``for_billing``); only the changed tokens use the estimate."""
+ billing = dataclasses.replace(BASE, read=5e-7)
+
+ def rates(model, *, long_context=False, provider=None, for_billing=False):
+ return billing if for_billing else BASE
+
+ monkeypatch.setattr(counterfactual, "resolve_rates", rates)
+ record = compute_savings_record(MEASURE, WARM, model="sonnet", provider="anthropic")
+ assert record.actual_input_cost_usd == pytest.approx(2_000 * IN + 90_000 * 5e-7 + 8_000 * WRITE)
+ assert record.saved_usd == pytest.approx(0.058618, abs=1e-6)
+
+
+def _fully_cached_original_of(original: int, *, model: str, provider: str):
+ """A 10,000-token sent body read wholly from cache, whose original was
+ ``original`` tokens: the counterfactual is that original alone."""
+ measure = ItemMeasure(
+ sent_tokens=10_000,
+ removed_old=original - 10_000,
+ removed_new=0,
+ sent_shared_prefix=10_000,
+ bust=0,
+ )
+ return compute_savings_record(measure, CacheMix(read=10_000), model=model, provider=provider)
+
+
+ ("original", "reads"),
+ [(1_023, 0), (1_024, 1_024), (1_151, 1_024), (1_152, 1_152), (2_000, 1_920)],
+)
+def test_an_openai_counterfactual_reads_whole_blocks_above_its_own_minimum(original, reads):
+ record = _fully_cached_original_of(original, model="gpt", provider="openai")
+ expected = reads * READ + (original - reads) * IN
+ assert record.counterfactual_input_cost_usd == pytest.approx(expected)
+
+
[email protected](("original", "reads"), [(4_095, 0), (4_096, 4_096), (4_097, 4_097)])
+def test_an_anthropic_counterfactual_obeys_its_models_cache_minimum(original, reads):
+ record = _fully_cached_original_of(original, model="haiku", provider="anthropic")
+ expected = reads * READ + (original - reads) * IN
+ assert record.counterfactual_input_cost_usd == pytest.approx(expected)
+
+
+def test_a_counterfactual_too_small_to_cache_pays_no_write_premium():
+ """Below its minimum the counterfactual can be neither read nor written."""
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0
+ )
+ usage = CacheMix(read=9_000, write_5m=1_000)
+ record = compute_savings_record(measure, usage, model="sonnet", provider="anthropic")
+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN)
+
+
+def test_a_grown_old_item_takes_its_reads_out_even_with_new_items():
+ """The new items keep the counterfactual above the minimum overall; its
+ 1,000-token old prefix is still too small to have been read."""
+ measure = ItemMeasure(
+ sent_tokens=30_000,
+ removed_old=-9_000,
+ removed_new=20_000,
+ sent_shared_prefix=10_000,
+ bust=0,
+ )
+ usage = CacheMix(read=9_000, uncached=21_000)
+ record = compute_savings_record(measure, usage, model="gpt", provider="openai")
+ assert record.counterfactual_input_cost_usd == pytest.approx(41_000 * IN)
+
+
+def test_a_catalog_minimum_the_provider_disproved_is_not_applied():
+ """Haiku's catalog says 4,096, but the provider read 3,000: keeping the
+ counterfactual's 1,500 reads is the conservative price."""
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=-5_000, removed_new=0, sent_shared_prefix=10_000, bust=0
+ )
+ usage = CacheMix(read=3_000, uncached=7_000)
+ record = compute_savings_record(measure, usage, model="haiku", provider="anthropic")
+ assert record.counterfactual_input_cost_usd == pytest.approx(1_500 * READ + 3_500 * IN)
+
+
+def test_nothing_removed_saves_nothing_whatever_the_read_alignment():
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=0, removed_new=0, sent_shared_prefix=2_000, bust=0
+ )
+ for provider, usage in (
+ ("openai", CacheMix(read=9_000, uncached=1_000)),
+ ("anthropic", CacheMix(read=900, write_5m=9_100)),
+ ):
+ record = compute_savings_record(measure, usage, model="haiku", provider=provider)
+ assert record.saved_usd == pytest.approx(0.0, abs=1e-12)
+
+
+def test_a_shrunk_bucket_is_never_priced_below_zero(monkeypatch):
+ """Reads a grown item gave up price at the billing rate they were billed
+ at, not at the estimate rate, so the counterfactual cannot go negative."""
+ billing = dataclasses.replace(BASE, read=1.5e-6)
+
+ def rates(model, *, long_context=False, provider=None, for_billing=False):
+ return billing if for_billing else BASE
+
+ monkeypatch.setattr(counterfactual, "resolve_rates", rates)
+ measure = ItemMeasure(
+ sent_tokens=10_000, removed_old=-9_000, removed_new=0, sent_shared_prefix=10_000, bust=0
+ )
+ record = compute_savings_record(
+ measure, CacheMix(read=9_000, uncached=1_000), model="gpt", provider="openai"
+ )
+ assert record.counterfactual_input_cost_usd == pytest.approx(1_000 * IN)
+
+
+def test_a_disproved_minimum_still_writes_nothing_smaller_than_was_cached():
+ """Haiku's catalog minimum is disproved by a 2,000-token read; a 500-token
+ counterfactual is below anything shown cacheable, so it writes nothing."""
+ measure = ItemMeasure(
+ sent_tokens=3_000, removed_old=-2_500, removed_new=0, sent_shared_prefix=3_000, bust=0
+ )
+ usage = CacheMix(read=2_000, write_5m=1_000)
+ record = compute_savings_record(measure, usage, model="haiku", provider="anthropic")
+ reads = 2_000 + -2_500 * (2 / 3)
+ assert record.counterfactual_input_cost_usd == pytest.approx(reads * READ + (500 - reads) * IN)
+
+
+def test_a_lost_baseline_prices_the_new_removal_at_the_read_floor():
+ """With the conversation's baseline evicted, the new items may be old ones
+ the provider had cached. Pricing them as new, at the miss rate, booked
+ $0.2236 here; the read rate, the floor of any input token, books $0.0152.
+ The 1,000 tokens Headroom added to a new item still cost the miss rate."""
+ lost = ItemMeasure(
+ sent_tokens=95_000,
+ removed_old=0,
+ removed_new=60_000 - 1_000,
+ sent_shared_prefix=0,
+ bust=0,
+ aligned=False,
+ baseline_lost=True,
+ grown_new=1_000,
+ )
+ k, p_miss = 100_000 / 95_000, 0.8 * WRITE + 0.2 * IN
+ as_new = compute_savings_record(
+ dataclasses.replace(lost, baseline_lost=False), WARM, model="sonnet", provider="anthropic"
+ )
+ assert as_new.saved_usd == pytest.approx(k * 59_000 * p_miss) # $0.2236
+
+ record = compute_savings_record(lost, WARM, model="sonnet", provider="anthropic")
+ assert record.saved_usd == pytest.approx(k * 60_000 * READ - k * 1_000 * p_miss) # $0.0152
+ assert record.saved_usd == pytest.approx(0.015158, abs=1e-6)
+ assert record.estimated
+ # Without usage too: list price for the rest, the read rate for the removal.
+ unbilled = compute_savings_record(lost, None, model="sonnet", provider="anthropic")
+ assert unbilled.saved_usd == pytest.approx(60_000 * READ - 1_000 * IN)