Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(pricing): reconcile cache billing with model-specific LiteLLM rates (#3033)
Abhay Singh463c1a43e30f
diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.pyindex 6a90205b3..c32168690 100644--- a/headroom/pricing/counterfactual.py+++ b/headroom/pricing/counterfactual.py@@ -55,6 +55,7 @@ beats precise and wrong.from __future__ import annotationsimport logging+import mathfrom dataclasses import dataclassfrom enum import Enumfrom functools import lru_cache@@ -338,6 +339,8 @@ class CacheRates:write_1h: floatuncached: floatbasis: str = BASIS_CATALOG+ read_is_catalog: bool = True+ write_is_catalog: bool = Truedef price(self, split: TokenSplit) -> float:return (@@ -373,15 +376,38 @@ def _litellm() -> Any | None:_RATE_CACHE_MAXSIZE = 256+def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context: bool) -> float:+ """Resolve an absent catalog slice through the model's billing calculator.++ LiteLLM versions and model adapters differ: missing rates can contribute+ zero, list price, or a model-specific discount. Never invent our own ratio.+ The result is cached with the other resolved rates, not probed per request.+ """+ tokens = LONG_CONTEXT_THRESHOLD_TOKENS + 1 if long_context else 1+ try:+ input_cost, _ = litellm.cost_per_token(+ model=model, prompt_tokens=tokens, completion_tokens=0, **{field: tokens}+ )+ rate = float(input_cost) / tokens+ except Exception:+ return 0.0+ return rate if math.isfinite(rate) and rate >= 0 else 0.0++@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)def resolve_rates(model: str,*,long_context: bool = False,provider: str | None = None,+ for_billing: bool = False,) -> CacheRates | None:"""Resolve per-bucket input rates for ``model``, or ``None`` if unpriceable.+ ``for_billing`` resolves absent cache rates through LiteLLM's model-specific+ billing calculation. Savings estimates use the default fallbacks below+ instead; those estimates must not manufacture billed cache charges.+Preference order, strongest first:1. **Catalog** — LiteLLM's per-model published rates, including the@@ -414,7 +440,8 @@ def resolve_rates(try:from headroom.pricing.litellm_pricing import resolve_litellm_model- info = litellm.model_cost.get(resolve_litellm_model(model), {}) or {}+ resolved_model = resolve_litellm_model(model)+ info = litellm.model_cost.get(resolved_model, {}) or {}except Exception:return None@@ -432,16 +459,33 @@ def resolve_rates("""Read ``field``, preferring its above-200k variant on long requests."""if long_context:hi = info.get(f"{field}_above_200k_tokens")- if hi:+ if hi is not None:return float(hi)value = info.get(field)return float(value) if value is not None else default- read = _tier("cache_read_input_token_cost", base)- # A provider that does not bill cache writes leaves this absent; writes then- # cost the same as ordinary input, which is exactly OpenAI's and Gemini's- # actual behaviour.- write_5m = _tier("cache_creation_input_token_cost", base)+ # Counterfactual savings retain their existing, labelled estimate fallback.+ # Billed slices consult the canonical model calculator below when the+ # catalog omits a rate; missing metadata does not imply free cache.+ missing_cache_rate = 0.0 if for_billing else base+ read = _tier("cache_read_input_token_cost", missing_cache_rate)+ # Savings estimates value otherwise unpriced writes at ordinary input;+ # billed provider-reported writes require their own catalog rate.+ write_5m = _tier("cache_creation_input_token_cost", missing_cache_rate)+ read_is_catalog = info.get("cache_read_input_token_cost") is not None or (+ long_context and info.get("cache_read_input_token_cost_above_200k_tokens") is not None+ )+ write_is_catalog = info.get("cache_creation_input_token_cost") is not None or (+ long_context and info.get("cache_creation_input_token_cost_above_200k_tokens") is not None+ )+ if for_billing and not read_is_catalog:+ read = _canonical_cache_rate(+ litellm, resolved_model, "cache_read_input_tokens", long_context=long_context+ )+ if for_billing and not write_is_catalog:+ write_5m = _canonical_cache_rate(+ litellm, resolved_model, "cache_creation_input_tokens", long_context=long_context+ )basis = BASIS_CATALOGwrite_1h_raw = info.get("cache_creation_input_token_cost_above_1hr")@@ -449,7 +493,7 @@ def resolve_rates(# No catalog publishes a combined 1h + above-200k rate, so the long# tier always derives. Ratio basis, and labelled as such.write_1h_raw = None- if write_1h_raw:+ if write_1h_raw is not None:write_1h = float(write_1h_raw)elif write_5m > base:# Real write premium present but no 1h rate: derive structurally.@@ -460,7 +504,14 @@ def resolve_rates(# price; a 1h write costs what any write costs.write_1h = write_5m- if read == base and write_5m == base and base > 0:+ if (+ not for_billing+ and not read_is_catalog+ and not write_is_catalog+ and read == base+ and write_5m == base+ and base > 0+ ):# Catalog priced the model but published no cache rates. Fall back to# the provider ratio table if we can identify the provider.econ = CACHE_ECONOMICS.get((provider or "").split(":")[-1].strip().lower())@@ -471,9 +522,19 @@ def resolve_rates(write_1h=base * float(econ["write_multiplier"]),uncached=base,basis=BASIS_PROVIDER_RATIO,+ read_is_catalog=read_is_catalog,+ write_is_catalog=write_is_catalog,)- return CacheRates(read=read, write_5m=write_5m, write_1h=write_1h, uncached=base, basis=basis)+ return CacheRates(+ read=read,+ write_5m=write_5m,+ write_1h=write_1h,+ uncached=base,+ basis=basis,+ read_is_catalog=read_is_catalog,+ write_is_catalog=write_is_catalog,+ )@dataclass(frozen=True)diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 3bf0e4979..af2e7a246 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -291,16 +291,25 @@ def build_prefix_cache_stats(price_per_1m = cost_tracker._get_list_price(model_name)if price_per_1m:input_price_per_token = price_per_1m / 1_000_000- cache_prices = cost_tracker._get_cache_prices(model_name)+ try:+ from headroom.pricing.counterfactual import resolve_rates++ cache_prices = resolve_rates(model_name)+ except Exception:+ cache_prices = Nonebest_tokens = tokens_sent# Catalog rates win; the provider ratio table is the fallback for a model# that publishes no cache pricing.pricing_source = "provider_default"if cache_prices and input_price_per_token:- _cr_price, _cw5_price, _cw1h_price, _uncached_price = cache_prices+ _cr_price = cache_prices.read+ _cw5_price = cache_prices.write_5m+ _cw1h_price = cache_prices.write_1h+ _uncached_price = cache_prices.uncachedif _uncached_price:- read_mult = _cr_price / _uncached_price+ if cache_prices.read_is_catalog:+ read_mult = _cr_price / _uncached_price# Blend the two write rates by the TTL mix this provider# actually wrote at, rather than charging every write the 5m# rate. Falls back to the 5m rate when nothing was written yet.@@ -309,8 +318,12 @@ def build_prefix_cache_stats(_w5 = max(0, _w_all - _w1h)_w_cost = _w5 * _cw5_price + _w1h * _cw1h_price_blended_write = (_w_cost / _w_all) if _w_all > 0 else _cw5_price- write_mult = _blended_write / _uncached_price- pricing_source = "catalog"+ if cache_prices.write_is_catalog:+ write_mult = _blended_write / _uncached_price+ if cache_prices.read_is_catalog and (cache_prices.write_is_catalog or not _w_all):+ pricing_source = "catalog"+ elif cache_prices.read_is_catalog or cache_prices.write_is_catalog:+ pricing_source = "mixed"# Calculate savings:# Cache reads save (1.0 - read_mult) per token vs uncached input price.@@ -1150,13 +1163,13 @@ class CostTracker:self._api_cache_read_by_model.get(model, 0) + cache_read_tokens)self._api_cache_write_by_model[model] = (- self._api_cache_write_by_model.get(model, 0) + cache_write_tokens+ self._api_cache_write_by_model.get(model, 0) + write_eff)self._api_cache_write_5m_by_model[model] = (- self._api_cache_write_5m_by_model.get(model, 0) + cache_write_5m_tokens+ self._api_cache_write_5m_by_model.get(model, 0) + write_5m_eff)self._api_cache_write_1h_by_model[model] = (- self._api_cache_write_1h_by_model.get(model, 0) + cache_write_1h_tokens+ self._api_cache_write_1h_by_model.get(model, 0) + write_1h_eff)self._api_uncached_by_model[model] = (self._api_uncached_by_model.get(model, 0) + uncached_tokens@@ -1401,7 +1414,7 @@ class CostTracker:return w5m * w5m_price + w1h * w1h_pricedef _get_cache_prices(- self, model: str, *, long_context: bool = False+ self, model: str, *, long_context: bool = False, for_billing: bool = True) -> tuple[float, float, float, float] | None:"""Per-token prices for (cache read, 5m write, 1h write, uncached input).@@ -1411,6 +1424,10 @@ class CostTracker:persisted tracker, and the durable ledger alike, so the three cannotdrift apart again.+ ``for_billing`` resolves missing cache catalog rates through LiteLLM's+ billing calculator. Savings consumers opt out to keep the resolver's+ counterfactual estimates.+The 1h write rate is new: LiteLLM publishes it per model as``cache_creation_input_token_cost_above_1hr`` (Sonnet: 2.00x base), andwhere a row omits it the structural multiplier in@@ -1427,7 +1444,7 @@ class CostTracker:try:from headroom.pricing.counterfactual import resolve_rates- rates = resolve_rates(model, long_context=long_context)+ rates = resolve_rates(model, long_context=long_context, for_billing=for_billing)except Exception:return Noneif rates is None or not rates.uncached:@@ -1586,7 +1603,7 @@ class CostTracker:| set(self._saved_list_by_tier)):model, long_context = key- prices = self._get_cache_prices(model, long_context=long_context)+ prices = self._get_cache_prices(model, long_context=long_context, for_billing=False)if not prices:continue_cr_price, cw5_price, cw1h_price, uncached_price = prices@@ -1622,7 +1639,7 @@ class CostTracker:| set(self._tool_saved_list_by_model))for model in tool_models:- prices = self._get_cache_prices(model)+ prices = self._get_cache_prices(model, for_billing=False)if not prices:continuecr_price, cw5_price, cw1h_price, uncached_price = pricesdiff --git a/tests/test_cost_cache_prices.py b/tests/test_cost_cache_prices.pynew file mode 100644index 000000000..8c999279c--- /dev/null+++ b/tests/test_cost_cache_prices.py@@ -0,0 +1,360 @@+"""Reconcile reported cache slices with model-specific LiteLLM billing.++Explicit catalog rates win independently. Missing rates use the canonical+calculator, whose fallback differs across supported versions and models;+unavailable calculations do not invent Headroom provider-wide prices.+Counterfactual savings and cache-card provenance remain separate from billing,+and inferred writes cannot double-charge the uncached input bucket.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from headroom.pricing.counterfactual import resolve_rates+from headroom.proxy.cost import CostTracker++[email protected](autouse=True)+def clear_rate_cache():+ resolve_rates.cache_clear()+ yield+ resolve_rates.cache_clear()+++def _patch_litellm(monkeypatch: pytest.MonkeyPatch, model_cost: dict) -> None:+ resolve_rates.cache_clear()+ monkeypatch.setattr(+ "headroom.pricing.counterfactual._litellm",+ lambda: SimpleNamespace(model_cost=model_cost),+ )+ monkeypatch.setattr(+ "headroom.pricing.litellm_pricing.resolve_litellm_model",+ lambda model: model,+ )+++class TestGetCachePrices:+ def test_uses_explicit_litellm_cache_fields_when_present(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "cache_read_input_token_cost": 1e-7,+ "cache_creation_input_token_cost": 1.25e-6,+ "litellm_provider": "anthropic",+ }+ },+ )+ assert CostTracker()._get_cache_prices("m") == (1e-7, 1.25e-6, 2e-6, 1e-6)++ def test_missing_cache_fields_priced_at_zero_not_full_rate(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ # A model with an input price but no cache fields: exactly the 1600++ # long-tail models the bug affects. Before the fix both slices billed at+ # the full uncached rate; now they fail closed at $0, reconciling with+ # what litellm.cost_per_token books for the same slice.+ _patch_litellm(+ monkeypatch,+ {"m": {"input_cost_per_token": 5e-7, "litellm_provider": "mistral"}},+ )+ assert CostTracker()._get_cache_prices("m") == (0.0, 0.0, 0.0, 5e-7)++ def test_only_the_missing_field_is_zeroed(self, monkeypatch: pytest.MonkeyPatch) -> None:+ # Explicit read cost present, write cost absent: keep the real read,+ # fail closed on the write.+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "cache_read_input_token_cost": 3e-7,+ "litellm_provider": "openai",+ }+ },+ )+ assert CostTracker()._get_cache_prices("m") == (3e-7, 0.0, 0.0, 1e-6)++ def test_no_input_price_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_litellm(monkeypatch, {"m": {"litellm_provider": "anthropic"}})+ assert CostTracker()._get_cache_prices("m") is None++ @pytest.mark.parametrize(+ "litellm_provider",+ ["bedrock", "bedrock_converse", "mistral", "fireworks_ai", "cohere_chat", None, "made-up"],+ )+ def test_heterogeneous_and_unknown_providers_do_not_inherit_anthropic_rates(+ self, monkeypatch: pytest.MonkeyPatch, litellm_provider+ ) -> None:+ # Regression: a missing cache field must never be back-filled with+ # Anthropic's 0.1x/1.25x economics. Bedrock fronts Anthropic, Amazon,+ # Meta, Mistral, Cohere and others; unknown OpenAI-compatible providers+ # are likewise not Anthropic-priced.+ uncached = 4e-6+ info = {"input_cost_per_token": uncached}+ if litellm_provider is not None:+ info["litellm_provider"] = litellm_provider+ _patch_litellm(monkeypatch, {"m": info})+ cache_read, cache_write, cache_write_1h, got_uncached = CostTracker()._get_cache_prices("m")+ assert got_uncached == uncached+ assert cache_read == 0.0+ assert cache_write == 0.0+ assert cache_write_1h == 0.0+ # Explicitly not the Anthropic multipliers the old fallback would apply.+ assert cache_read != pytest.approx(uncached * 0.1)+ assert cache_write != pytest.approx(uncached * 1.25)++ def test_missing_cache_slices_match_canonical_zero_pricing(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """The missing cache slices are priced at $0, the same amount the canonical+ litellm.cost_per_token path books for a slice LiteLLM cannot price, so the+ cache breakdown reconciles with the primary accounting instead of diverging.+ """+ _patch_litellm(+ monkeypatch,+ {"m": {"input_cost_per_token": 5e-7, "litellm_provider": "fireworks_ai"}},+ )+ cr_price, cw_price, _, uncached_price = CostTracker()._get_cache_prices("m")+ cr_tokens, cw_tokens = 20_000, 5_000+ # Cache-slice cost contributed to totals() is exactly $0, matching canonical.+ assert cr_tokens * cr_price + cw_tokens * cw_price == 0.0+ assert uncached_price == 5e-7++ def test_totals_and_stats_charge_only_known_slices(self, monkeypatch):+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 5e-7}})+ tracker = CostTracker()+ tracker.record_tokens(+ model="m",+ tokens_saved=0,+ tokens_sent=26_000,+ cache_read_tokens=20_000,+ cache_write_tokens=5_000,+ uncached_tokens=1_000,+ )+ assert tracker.totals() == (26_000, 0.0005)+ assert tracker.stats()["total_input_cost_usd"] == 0.0005++ def test_missing_read_preserves_explicit_write_and_ttl_tier(self, monkeypatch):+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "input_cost_per_token_above_200k_tokens": 2e-6,+ "cache_creation_input_token_cost": 1.25e-6,+ "cache_creation_input_token_cost_above_200k_tokens": 2.5e-6,+ "cache_creation_input_token_cost_above_1hr": 2e-6,+ }+ },+ )+ assert CostTracker()._get_cache_prices("m") == (0.0, 1.25e-6, 2e-6, 1e-6)+ assert CostTracker()._get_cache_prices("m", long_context=True) == (+ 0.0,+ 2.5e-6,+ 4e-6,+ 2e-6,+ )++ def test_savings_provider_ratio_remains_separate_from_billed_prices(self, monkeypatch):+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})+ assert resolve_rates("m", provider="anthropic").read == 1e-7+ assert CostTracker()._get_cache_prices("m")[0] == 0.0++ def test_missing_write_price_keeps_live_zone_and_tool_savings(self, monkeypatch):+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})+ tracker = CostTracker()+ tracker.record_tokens(+ model="m",+ tokens_saved=10_000,+ tokens_sent=10_000,+ cache_write_tokens=10_000,+ tool_schema_saved=5_000,+ )+ stats = tracker.stats()+ assert stats["total_input_cost_usd"] == 0.0+ assert stats["cache_aware_savings_usd"] == pytest.approx(0.01)+ assert stats["tool_savings_usd"] == pytest.approx(0.005)++ @pytest.mark.parametrize("cache_read_price", [None, "invalid"])+ def test_unknown_read_rate_is_not_advertised_as_catalog_free_cache(+ self, monkeypatch, cache_read_price+ ):+ from headroom.proxy.cost import build_prefix_cache_stats+ from headroom.proxy.prometheus_metrics import PrometheusMetrics++ _patch_litellm(+ monkeypatch,+ {+ "gpt-unknown": {+ "input_cost_per_token": 1e-6,+ "cache_read_input_token_cost": cache_read_price,+ }+ },+ )+ tracker = CostTracker()+ tracker._tokens_sent_by_model["gpt-unknown"] = 1_000_000+ monkeypatch.setattr(tracker, "_get_list_price", lambda model: 1.0)+ metrics = PrometheusMetrics()+ metrics.cache_by_provider["openai"].update(+ {+ "requests": 1,+ "hit_requests": 1,+ "cache_read_tokens": 1_000_000,+ }+ )+ row = build_prefix_cache_stats(metrics, tracker)["by_provider"]["openai"]+ assert row["cache_pricing_source"] == "provider_default"+ assert row["savings_usd"] == pytest.approx(0.5)++ def test_canonical_litellm_cache_only_cost_matches_dashboard(self, monkeypatch):+ import litellm++ from headroom.pricing.litellm_pricing import resolve_litellm_model++ # Keep the actual canonical calculator, with a deterministic catalog+ # row: the shared test suite enriches real model rows with cache rates.+ model = "openai/headroom-missing-cache-rate-regression"+ monkeypatch.setitem(+ litellm.model_cost,+ model,+ {+ "input_cost_per_token": 5e-7,+ "output_cost_per_token": 1e-6,+ "litellm_provider": "openai",+ "mode": "chat",+ },+ )+ resolved = resolve_litellm_model(model)+ info = litellm.model_cost[resolved]+ assert info.get("cache_read_input_token_cost") is None+ assert info.get("cache_creation_input_token_cost") is None+ canonical, _ = litellm.cost_per_token(+ model=resolved,+ prompt_tokens=10_000,+ completion_tokens=0,+ cache_read_input_tokens=10_000,+ )+ tracker = CostTracker()+ tracker.record_tokens(+ model=model, tokens_saved=0, tokens_sent=10_000, cache_read_tokens=10_000+ )+ assert tracker.totals()[1] == round(canonical, 4)++ @pytest.mark.parametrize("long_context", [False, True])+ def test_missing_catalog_rates_use_the_model_specific_canonical_calculation(+ self, monkeypatch, long_context+ ):+ resolve_rates.cache_clear()+ monkeypatch.setattr(+ "headroom.pricing.litellm_pricing.resolve_litellm_model", lambda model: model+ )+ calls = []++ def canonical(**kwargs):+ calls.append(kwargs)+ factor = 2 if kwargs["prompt_tokens"] > 200_000 else 1+ return (+ factor+ * (+ kwargs.get("cache_read_input_tokens", 0) * 4e-7+ + kwargs.get("cache_creation_input_tokens", 0) * 1e-6+ ),+ 0.0,+ )++ monkeypatch.setattr(+ "headroom.pricing.counterfactual._litellm",+ lambda: SimpleNamespace(+ model_cost={+ "m": {+ "input_cost_per_token": 1e-6,+ "input_cost_per_token_above_200k_tokens": 2e-6,+ }+ },+ cost_per_token=canonical,+ ),+ )+ factor = 2 if long_context else 1+ assert CostTracker()._get_cache_prices("m", long_context=long_context) == (+ 4e-7 * factor,+ 1e-6 * factor,+ 1e-6 * factor,+ 1e-6 * factor,+ )+ assert len(calls) == 2+ assert all(+ call["model"] == "m" and call["prompt_tokens"] == (200_001 if long_context else 1)+ for call in calls+ )++ def test_inferred_writes_are_not_billed_twice_with_canonical_rates(self, monkeypatch):+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "cache_read_input_token_cost": 5e-7,+ "cache_creation_input_token_cost": 1e-6,+ }+ },+ )+ tracker = CostTracker()+ tracker.record_tokens(+ model="m",+ tokens_saved=0,+ tokens_sent=20_000,+ cache_read_tokens=10_000,+ cache_write_tokens=10_000,+ cache_write_5m_tokens=10_000,+ uncached_tokens=10_000,+ cache_inferred=True,+ )+ assert tracker.totals()[1] == pytest.approx(0.015)+ assert tracker.stats()["total_input_cost_usd"] == pytest.approx(0.015)+ assert tracker._api_cache_write_by_model["m"] == 0+ assert tracker._api_cache_write_5m_by_model["m"] == 0++ def test_explicit_zero_cache_tier_and_one_hour_prices_are_authoritative(self, monkeypatch):+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "input_cost_per_token_above_200k_tokens": 2e-6,+ "cache_read_input_token_cost": 1e-7,+ "cache_read_input_token_cost_above_200k_tokens": 0.0,+ "cache_creation_input_token_cost": 1.25e-6,+ "cache_creation_input_token_cost_above_200k_tokens": 0.0,+ "cache_creation_input_token_cost_above_1hr": 0.0,+ }+ },+ )+ assert CostTracker()._get_cache_prices("m")[2] == 0.0+ assert CostTracker()._get_cache_prices("m", long_context=True) == (+ 0.0,+ 0.0,+ 0.0,+ 2e-6,+ )++ def test_explicit_list_rate_cache_prices_do_not_inherit_provider_discounts(self, monkeypatch):+ _patch_litellm(+ monkeypatch,+ {+ "m": {+ "input_cost_per_token": 1e-6,+ "cache_read_input_token_cost": 1e-6,+ "cache_creation_input_token_cost": 1e-6,+ }+ },+ )+ rates = resolve_rates("m", provider="anthropic")+ assert rates.read == rates.write_5m == 1e-6+ assert rates.read_is_catalog and rates.write_is_catalog