Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(pricing): reconcile cache billing with model-specific LiteLLM rates (#3033)

Abhay Singh463c1a43e30f
diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.py
index 6a90205b3..c32168690 100644
--- a/headroom/pricing/counterfactual.py
+++ b/headroom/pricing/counterfactual.py
@@ -55,6 +55,7 @@ beats precise and wrong.
from __future__ import annotations
import logging
+import math
from dataclasses import dataclass
from enum import Enum
from functools import lru_cache
@@ -338,6 +339,8 @@ class CacheRates:
write_1h: float
uncached: float
basis: str = BASIS_CATALOG
+ read_is_catalog: bool = True
+ write_is_catalog: bool = True
def price(self, split: TokenSplit) -> float:
return (
@@ -373,15 +376,38 @@ def _litellm() -> Any | None:
_RATE_CACHE_MAXSIZE = 256
+def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context: bool) -> float:
+ """Resolve an absent catalog slice through the model's billing calculator.
+
+ LiteLLM versions and model adapters differ: missing rates can contribute
+ zero, list price, or a model-specific discount. Never invent our own ratio.
+ The result is cached with the other resolved rates, not probed per request.
+ """
+ tokens = LONG_CONTEXT_THRESHOLD_TOKENS + 1 if long_context else 1
+ try:
+ input_cost, _ = litellm.cost_per_token(
+ model=model, prompt_tokens=tokens, completion_tokens=0, **{field: tokens}
+ )
+ rate = float(input_cost) / tokens
+ except Exception:
+ return 0.0
+ return rate if math.isfinite(rate) and rate >= 0 else 0.0
+
+
@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)
def resolve_rates(
model: str,
*,
long_context: bool = False,
provider: str | None = None,
+ for_billing: bool = False,
) -> CacheRates | None:
"""Resolve per-bucket input rates for ``model``, or ``None`` if unpriceable.
+ ``for_billing`` resolves absent cache rates through LiteLLM's model-specific
+ billing calculation. Savings estimates use the default fallbacks below
+ instead; those estimates must not manufacture billed cache charges.
+
Preference order, strongest first:
1. **Catalog** — LiteLLM's per-model published rates, including the
@@ -414,7 +440,8 @@ def resolve_rates(
try:
from headroom.pricing.litellm_pricing import resolve_litellm_model
- info = litellm.model_cost.get(resolve_litellm_model(model), {}) or {}
+ resolved_model = resolve_litellm_model(model)
+ info = litellm.model_cost.get(resolved_model, {}) or {}
except Exception:
return None
@@ -432,16 +459,33 @@ def resolve_rates(
"""Read ``field``, preferring its above-200k variant on long requests."""
if long_context:
hi = info.get(f"{field}_above_200k_tokens")
- if hi:
+ if hi is not None:
return float(hi)
value = info.get(field)
return float(value) if value is not None else default
- read = _tier("cache_read_input_token_cost", base)
- # A provider that does not bill cache writes leaves this absent; writes then
- # cost the same as ordinary input, which is exactly OpenAI's and Gemini's
- # actual behaviour.
- write_5m = _tier("cache_creation_input_token_cost", base)
+ # Counterfactual savings retain their existing, labelled estimate fallback.
+ # Billed slices consult the canonical model calculator below when the
+ # catalog omits a rate; missing metadata does not imply free cache.
+ missing_cache_rate = 0.0 if for_billing else base
+ read = _tier("cache_read_input_token_cost", missing_cache_rate)
+ # Savings estimates value otherwise unpriced writes at ordinary input;
+ # billed provider-reported writes require their own catalog rate.
+ write_5m = _tier("cache_creation_input_token_cost", missing_cache_rate)
+ read_is_catalog = info.get("cache_read_input_token_cost") is not None or (
+ long_context and info.get("cache_read_input_token_cost_above_200k_tokens") is not None
+ )
+ write_is_catalog = info.get("cache_creation_input_token_cost") is not None or (
+ long_context and info.get("cache_creation_input_token_cost_above_200k_tokens") is not None
+ )
+ if for_billing and not read_is_catalog:
+ read = _canonical_cache_rate(
+ litellm, resolved_model, "cache_read_input_tokens", long_context=long_context
+ )
+ if for_billing and not write_is_catalog:
+ write_5m = _canonical_cache_rate(
+ litellm, resolved_model, "cache_creation_input_tokens", long_context=long_context
+ )
basis = BASIS_CATALOG
write_1h_raw = info.get("cache_creation_input_token_cost_above_1hr")
@@ -449,7 +493,7 @@ def resolve_rates(
# No catalog publishes a combined 1h + above-200k rate, so the long
# tier always derives. Ratio basis, and labelled as such.
write_1h_raw = None
- if write_1h_raw:
+ if write_1h_raw is not None:
write_1h = float(write_1h_raw)
elif write_5m > base:
# Real write premium present but no 1h rate: derive structurally.
@@ -460,7 +504,14 @@ def resolve_rates(
# price; a 1h write costs what any write costs.
write_1h = write_5m
- if read == base and write_5m == base and base > 0:
+ if (
+ not for_billing
+ and not read_is_catalog
+ and not write_is_catalog
+ and read == base
+ and write_5m == base
+ and base > 0
+ ):
# Catalog priced the model but published no cache rates. Fall back to
# the provider ratio table if we can identify the provider.
econ = CACHE_ECONOMICS.get((provider or "").split(":")[-1].strip().lower())
@@ -471,9 +522,19 @@ def resolve_rates(
write_1h=base * float(econ["write_multiplier"]),
uncached=base,
basis=BASIS_PROVIDER_RATIO,
+ read_is_catalog=read_is_catalog,
+ write_is_catalog=write_is_catalog,
)
- return CacheRates(read=read, write_5m=write_5m, write_1h=write_1h, uncached=base, basis=basis)
+ return CacheRates(
+ read=read,
+ write_5m=write_5m,
+ write_1h=write_1h,
+ uncached=base,
+ basis=basis,
+ read_is_catalog=read_is_catalog,
+ write_is_catalog=write_is_catalog,
+ )
@dataclass(frozen=True)
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 3bf0e4979..af2e7a246 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -291,16 +291,25 @@ def build_prefix_cache_stats(
price_per_1m = cost_tracker._get_list_price(model_name)
if price_per_1m:
input_price_per_token = price_per_1m / 1_000_000
- cache_prices = cost_tracker._get_cache_prices(model_name)
+ try:
+ from headroom.pricing.counterfactual import resolve_rates
+
+ cache_prices = resolve_rates(model_name)
+ except Exception:
+ cache_prices = None
best_tokens = tokens_sent
# Catalog rates win; the provider ratio table is the fallback for a model
# that publishes no cache pricing.
pricing_source = "provider_default"
if cache_prices and input_price_per_token:
- _cr_price, _cw5_price, _cw1h_price, _uncached_price = cache_prices
+ _cr_price = cache_prices.read
+ _cw5_price = cache_prices.write_5m
+ _cw1h_price = cache_prices.write_1h
+ _uncached_price = cache_prices.uncached
if _uncached_price:
- read_mult = _cr_price / _uncached_price
+ if cache_prices.read_is_catalog:
+ read_mult = _cr_price / _uncached_price
# Blend the two write rates by the TTL mix this provider
# actually wrote at, rather than charging every write the 5m
# rate. Falls back to the 5m rate when nothing was written yet.
@@ -309,8 +318,12 @@ def build_prefix_cache_stats(
_w5 = max(0, _w_all - _w1h)
_w_cost = _w5 * _cw5_price + _w1h * _cw1h_price
_blended_write = (_w_cost / _w_all) if _w_all > 0 else _cw5_price
- write_mult = _blended_write / _uncached_price
- pricing_source = "catalog"
+ if cache_prices.write_is_catalog:
+ write_mult = _blended_write / _uncached_price
+ if cache_prices.read_is_catalog and (cache_prices.write_is_catalog or not _w_all):
+ pricing_source = "catalog"
+ elif cache_prices.read_is_catalog or cache_prices.write_is_catalog:
+ pricing_source = "mixed"
# Calculate savings:
# Cache reads save (1.0 - read_mult) per token vs uncached input price.
@@ -1150,13 +1163,13 @@ class CostTracker:
self._api_cache_read_by_model.get(model, 0) + cache_read_tokens
)
self._api_cache_write_by_model[model] = (
- self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
+ self._api_cache_write_by_model.get(model, 0) + write_eff
)
self._api_cache_write_5m_by_model[model] = (
- self._api_cache_write_5m_by_model.get(model, 0) + cache_write_5m_tokens
+ self._api_cache_write_5m_by_model.get(model, 0) + write_5m_eff
)
self._api_cache_write_1h_by_model[model] = (
- self._api_cache_write_1h_by_model.get(model, 0) + cache_write_1h_tokens
+ self._api_cache_write_1h_by_model.get(model, 0) + write_1h_eff
)
self._api_uncached_by_model[model] = (
self._api_uncached_by_model.get(model, 0) + uncached_tokens
@@ -1401,7 +1414,7 @@ class CostTracker:
return w5m * w5m_price + w1h * w1h_price
def _get_cache_prices(
- self, model: str, *, long_context: bool = False
+ self, model: str, *, long_context: bool = False, for_billing: bool = True
) -> tuple[float, float, float, float] | None:
"""Per-token prices for (cache read, 5m write, 1h write, uncached input).
@@ -1411,6 +1424,10 @@ class CostTracker:
persisted tracker, and the durable ledger alike, so the three cannot
drift apart again.
+ ``for_billing`` resolves missing cache catalog rates through LiteLLM's
+ billing calculator. Savings consumers opt out to keep the resolver's
+ counterfactual estimates.
+
The 1h write rate is new: LiteLLM publishes it per model as
``cache_creation_input_token_cost_above_1hr`` (Sonnet: 2.00x base), and
where a row omits it the structural multiplier in
@@ -1427,7 +1444,7 @@ class CostTracker:
try:
from headroom.pricing.counterfactual import resolve_rates
- rates = resolve_rates(model, long_context=long_context)
+ rates = resolve_rates(model, long_context=long_context, for_billing=for_billing)
except Exception:
return None
if rates is None or not rates.uncached:
@@ -1586,7 +1603,7 @@ class CostTracker:
| set(self._saved_list_by_tier)
):
model, long_context = key
- prices = self._get_cache_prices(model, long_context=long_context)
+ prices = self._get_cache_prices(model, long_context=long_context, for_billing=False)
if not prices:
continue
_cr_price, cw5_price, cw1h_price, uncached_price = prices
@@ -1622,7 +1639,7 @@ class CostTracker:
| set(self._tool_saved_list_by_model)
)
for model in tool_models:
- prices = self._get_cache_prices(model)
+ prices = self._get_cache_prices(model, for_billing=False)
if not prices:
continue
cr_price, cw5_price, cw1h_price, uncached_price = prices
diff --git a/tests/test_cost_cache_prices.py b/tests/test_cost_cache_prices.py
new file mode 100644
index 000000000..8c999279c
--- /dev/null
+++ b/tests/test_cost_cache_prices.py
@@ -0,0 +1,360 @@
+"""Reconcile reported cache slices with model-specific LiteLLM billing.
+
+Explicit catalog rates win independently. Missing rates use the canonical
+calculator, whose fallback differs across supported versions and models;
+unavailable calculations do not invent Headroom provider-wide prices.
+Counterfactual savings and cache-card provenance remain separate from billing,
+and inferred writes cannot double-charge the uncached input bucket.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from headroom.pricing.counterfactual import resolve_rates
+from headroom.proxy.cost import CostTracker
+
+
[email protected](autouse=True)
+def clear_rate_cache():
+ resolve_rates.cache_clear()
+ yield
+ resolve_rates.cache_clear()
+
+
+def _patch_litellm(monkeypatch: pytest.MonkeyPatch, model_cost: dict) -> None:
+ resolve_rates.cache_clear()
+ monkeypatch.setattr(
+ "headroom.pricing.counterfactual._litellm",
+ lambda: SimpleNamespace(model_cost=model_cost),
+ )
+ monkeypatch.setattr(
+ "headroom.pricing.litellm_pricing.resolve_litellm_model",
+ lambda model: model,
+ )
+
+
+class TestGetCachePrices:
+ def test_uses_explicit_litellm_cache_fields_when_present(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "cache_read_input_token_cost": 1e-7,
+ "cache_creation_input_token_cost": 1.25e-6,
+ "litellm_provider": "anthropic",
+ }
+ },
+ )
+ assert CostTracker()._get_cache_prices("m") == (1e-7, 1.25e-6, 2e-6, 1e-6)
+
+ def test_missing_cache_fields_priced_at_zero_not_full_rate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # A model with an input price but no cache fields: exactly the 1600+
+ # long-tail models the bug affects. Before the fix both slices billed at
+ # the full uncached rate; now they fail closed at $0, reconciling with
+ # what litellm.cost_per_token books for the same slice.
+ _patch_litellm(
+ monkeypatch,
+ {"m": {"input_cost_per_token": 5e-7, "litellm_provider": "mistral"}},
+ )
+ assert CostTracker()._get_cache_prices("m") == (0.0, 0.0, 0.0, 5e-7)
+
+ def test_only_the_missing_field_is_zeroed(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # Explicit read cost present, write cost absent: keep the real read,
+ # fail closed on the write.
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "cache_read_input_token_cost": 3e-7,
+ "litellm_provider": "openai",
+ }
+ },
+ )
+ assert CostTracker()._get_cache_prices("m") == (3e-7, 0.0, 0.0, 1e-6)
+
+ def test_no_input_price_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _patch_litellm(monkeypatch, {"m": {"litellm_provider": "anthropic"}})
+ assert CostTracker()._get_cache_prices("m") is None
+
+ @pytest.mark.parametrize(
+ "litellm_provider",
+ ["bedrock", "bedrock_converse", "mistral", "fireworks_ai", "cohere_chat", None, "made-up"],
+ )
+ def test_heterogeneous_and_unknown_providers_do_not_inherit_anthropic_rates(
+ self, monkeypatch: pytest.MonkeyPatch, litellm_provider
+ ) -> None:
+ # Regression: a missing cache field must never be back-filled with
+ # Anthropic's 0.1x/1.25x economics. Bedrock fronts Anthropic, Amazon,
+ # Meta, Mistral, Cohere and others; unknown OpenAI-compatible providers
+ # are likewise not Anthropic-priced.
+ uncached = 4e-6
+ info = {"input_cost_per_token": uncached}
+ if litellm_provider is not None:
+ info["litellm_provider"] = litellm_provider
+ _patch_litellm(monkeypatch, {"m": info})
+ cache_read, cache_write, cache_write_1h, got_uncached = CostTracker()._get_cache_prices("m")
+ assert got_uncached == uncached
+ assert cache_read == 0.0
+ assert cache_write == 0.0
+ assert cache_write_1h == 0.0
+ # Explicitly not the Anthropic multipliers the old fallback would apply.
+ assert cache_read != pytest.approx(uncached * 0.1)
+ assert cache_write != pytest.approx(uncached * 1.25)
+
+ def test_missing_cache_slices_match_canonical_zero_pricing(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """The missing cache slices are priced at $0, the same amount the canonical
+ litellm.cost_per_token path books for a slice LiteLLM cannot price, so the
+ cache breakdown reconciles with the primary accounting instead of diverging.
+ """
+ _patch_litellm(
+ monkeypatch,
+ {"m": {"input_cost_per_token": 5e-7, "litellm_provider": "fireworks_ai"}},
+ )
+ cr_price, cw_price, _, uncached_price = CostTracker()._get_cache_prices("m")
+ cr_tokens, cw_tokens = 20_000, 5_000
+ # Cache-slice cost contributed to totals() is exactly $0, matching canonical.
+ assert cr_tokens * cr_price + cw_tokens * cw_price == 0.0
+ assert uncached_price == 5e-7
+
+ def test_totals_and_stats_charge_only_known_slices(self, monkeypatch):
+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 5e-7}})
+ tracker = CostTracker()
+ tracker.record_tokens(
+ model="m",
+ tokens_saved=0,
+ tokens_sent=26_000,
+ cache_read_tokens=20_000,
+ cache_write_tokens=5_000,
+ uncached_tokens=1_000,
+ )
+ assert tracker.totals() == (26_000, 0.0005)
+ assert tracker.stats()["total_input_cost_usd"] == 0.0005
+
+ def test_missing_read_preserves_explicit_write_and_ttl_tier(self, monkeypatch):
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "input_cost_per_token_above_200k_tokens": 2e-6,
+ "cache_creation_input_token_cost": 1.25e-6,
+ "cache_creation_input_token_cost_above_200k_tokens": 2.5e-6,
+ "cache_creation_input_token_cost_above_1hr": 2e-6,
+ }
+ },
+ )
+ assert CostTracker()._get_cache_prices("m") == (0.0, 1.25e-6, 2e-6, 1e-6)
+ assert CostTracker()._get_cache_prices("m", long_context=True) == (
+ 0.0,
+ 2.5e-6,
+ 4e-6,
+ 2e-6,
+ )
+
+ def test_savings_provider_ratio_remains_separate_from_billed_prices(self, monkeypatch):
+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})
+ assert resolve_rates("m", provider="anthropic").read == 1e-7
+ assert CostTracker()._get_cache_prices("m")[0] == 0.0
+
+ def test_missing_write_price_keeps_live_zone_and_tool_savings(self, monkeypatch):
+ _patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})
+ tracker = CostTracker()
+ tracker.record_tokens(
+ model="m",
+ tokens_saved=10_000,
+ tokens_sent=10_000,
+ cache_write_tokens=10_000,
+ tool_schema_saved=5_000,
+ )
+ stats = tracker.stats()
+ assert stats["total_input_cost_usd"] == 0.0
+ assert stats["cache_aware_savings_usd"] == pytest.approx(0.01)
+ assert stats["tool_savings_usd"] == pytest.approx(0.005)
+
+ @pytest.mark.parametrize("cache_read_price", [None, "invalid"])
+ def test_unknown_read_rate_is_not_advertised_as_catalog_free_cache(
+ self, monkeypatch, cache_read_price
+ ):
+ from headroom.proxy.cost import build_prefix_cache_stats
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
+
+ _patch_litellm(
+ monkeypatch,
+ {
+ "gpt-unknown": {
+ "input_cost_per_token": 1e-6,
+ "cache_read_input_token_cost": cache_read_price,
+ }
+ },
+ )
+ tracker = CostTracker()
+ tracker._tokens_sent_by_model["gpt-unknown"] = 1_000_000
+ monkeypatch.setattr(tracker, "_get_list_price", lambda model: 1.0)
+ metrics = PrometheusMetrics()
+ metrics.cache_by_provider["openai"].update(
+ {
+ "requests": 1,
+ "hit_requests": 1,
+ "cache_read_tokens": 1_000_000,
+ }
+ )
+ row = build_prefix_cache_stats(metrics, tracker)["by_provider"]["openai"]
+ assert row["cache_pricing_source"] == "provider_default"
+ assert row["savings_usd"] == pytest.approx(0.5)
+
+ def test_canonical_litellm_cache_only_cost_matches_dashboard(self, monkeypatch):
+ import litellm
+
+ from headroom.pricing.litellm_pricing import resolve_litellm_model
+
+ # Keep the actual canonical calculator, with a deterministic catalog
+ # row: the shared test suite enriches real model rows with cache rates.
+ model = "openai/headroom-missing-cache-rate-regression"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {
+ "input_cost_per_token": 5e-7,
+ "output_cost_per_token": 1e-6,
+ "litellm_provider": "openai",
+ "mode": "chat",
+ },
+ )
+ resolved = resolve_litellm_model(model)
+ info = litellm.model_cost[resolved]
+ assert info.get("cache_read_input_token_cost") is None
+ assert info.get("cache_creation_input_token_cost") is None
+ canonical, _ = litellm.cost_per_token(
+ model=resolved,
+ prompt_tokens=10_000,
+ completion_tokens=0,
+ cache_read_input_tokens=10_000,
+ )
+ tracker = CostTracker()
+ tracker.record_tokens(
+ model=model, tokens_saved=0, tokens_sent=10_000, cache_read_tokens=10_000
+ )
+ assert tracker.totals()[1] == round(canonical, 4)
+
+ @pytest.mark.parametrize("long_context", [False, True])
+ def test_missing_catalog_rates_use_the_model_specific_canonical_calculation(
+ self, monkeypatch, long_context
+ ):
+ resolve_rates.cache_clear()
+ monkeypatch.setattr(
+ "headroom.pricing.litellm_pricing.resolve_litellm_model", lambda model: model
+ )
+ calls = []
+
+ def canonical(**kwargs):
+ calls.append(kwargs)
+ factor = 2 if kwargs["prompt_tokens"] > 200_000 else 1
+ return (
+ factor
+ * (
+ kwargs.get("cache_read_input_tokens", 0) * 4e-7
+ + kwargs.get("cache_creation_input_tokens", 0) * 1e-6
+ ),
+ 0.0,
+ )
+
+ monkeypatch.setattr(
+ "headroom.pricing.counterfactual._litellm",
+ lambda: SimpleNamespace(
+ model_cost={
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "input_cost_per_token_above_200k_tokens": 2e-6,
+ }
+ },
+ cost_per_token=canonical,
+ ),
+ )
+ factor = 2 if long_context else 1
+ assert CostTracker()._get_cache_prices("m", long_context=long_context) == (
+ 4e-7 * factor,
+ 1e-6 * factor,
+ 1e-6 * factor,
+ 1e-6 * factor,
+ )
+ assert len(calls) == 2
+ assert all(
+ call["model"] == "m" and call["prompt_tokens"] == (200_001 if long_context else 1)
+ for call in calls
+ )
+
+ def test_inferred_writes_are_not_billed_twice_with_canonical_rates(self, monkeypatch):
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "cache_read_input_token_cost": 5e-7,
+ "cache_creation_input_token_cost": 1e-6,
+ }
+ },
+ )
+ tracker = CostTracker()
+ tracker.record_tokens(
+ model="m",
+ tokens_saved=0,
+ tokens_sent=20_000,
+ cache_read_tokens=10_000,
+ cache_write_tokens=10_000,
+ cache_write_5m_tokens=10_000,
+ uncached_tokens=10_000,
+ cache_inferred=True,
+ )
+ assert tracker.totals()[1] == pytest.approx(0.015)
+ assert tracker.stats()["total_input_cost_usd"] == pytest.approx(0.015)
+ assert tracker._api_cache_write_by_model["m"] == 0
+ assert tracker._api_cache_write_5m_by_model["m"] == 0
+
+ def test_explicit_zero_cache_tier_and_one_hour_prices_are_authoritative(self, monkeypatch):
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "input_cost_per_token_above_200k_tokens": 2e-6,
+ "cache_read_input_token_cost": 1e-7,
+ "cache_read_input_token_cost_above_200k_tokens": 0.0,
+ "cache_creation_input_token_cost": 1.25e-6,
+ "cache_creation_input_token_cost_above_200k_tokens": 0.0,
+ "cache_creation_input_token_cost_above_1hr": 0.0,
+ }
+ },
+ )
+ assert CostTracker()._get_cache_prices("m")[2] == 0.0
+ assert CostTracker()._get_cache_prices("m", long_context=True) == (
+ 0.0,
+ 0.0,
+ 0.0,
+ 2e-6,
+ )
+
+ def test_explicit_list_rate_cache_prices_do_not_inherit_provider_discounts(self, monkeypatch):
+ _patch_litellm(
+ monkeypatch,
+ {
+ "m": {
+ "input_cost_per_token": 1e-6,
+ "cache_read_input_token_cost": 1e-6,
+ "cache_creation_input_token_cost": 1e-6,
+ }
+ },
+ )
+ rates = resolve_rates("m", provider="anthropic")
+ assert rates.read == rates.write_5m == 1e-6
+ assert rates.read_is_catalog and rates.write_is_catalog