Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(pricing): keep the long-context premium when litellm drops the 200k rate (#4109)

Tejas Chopra8c1beb20aeb8
diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.py
index 7634f3ccc..e8b29eed4 100644
--- a/headroom/providers/anthropic.py
+++ b/headroom/providers/anthropic.py
@@ -30,7 +30,8 @@ from headroom.pricing.deepseek_tiers import (
OFF_PEAK_RATES_PER_1M,
off_peak_rates,
)
-from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
+from headroom.pricing.litellm_model_resolution import pricing_lookup_candidates
+from headroom.pricing.litellm_pricing import estimate_cost_from_tokens, get_litellm_model_cost
from headroom.tokenizers.base import (
TokenCountCache,
coerce_countable_text,
@@ -219,8 +220,9 @@ def _apply_long_context_premium(
) -> dict[str, float]:
"""Return ``pricing`` scaled by the long-context premium where it applies.
- Used only on the manual fallback path; the LiteLLM path already applies the
- published above-threshold rates itself.
+ Used on the manual path, which also serves long-context requests whose
+ LiteLLM entry has no above-threshold rate (see
+ ``_litellm_lacks_long_context_rate``).
"""
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
return pricing
@@ -229,6 +231,26 @@ def _apply_long_context_premium(
return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}
+def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
+ """True when this request needs the long-context premium but LiteLLM has no rate for it.
+
+ LiteLLM's live price map has dropped ``input_cost_per_token_above_200k_tokens``
+ from the bare Anthropic ids before, and pricing such a request through it then
+ bills the base tier: about half the real cost of a long session. The manual
+ table still carries the premium, so the caller uses it instead.
+ """
+ if input_tokens <= _LONG_CONTEXT_THRESHOLD:
+ return False
+ if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
+ return False
+ cost_data = get_litellm_model_cost()
+ for candidate in pricing_lookup_candidates(model):
+ info = cost_data.get(candidate)
+ if isinstance(info, dict):
+ return "input_cost_per_token_above_200k_tokens" not in info
+ return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
+
+
# Default limits for pattern-based inference
# Used when a model isn't in the explicit list but matches a known pattern
_PATTERN_DEFAULTS = {
@@ -837,16 +859,18 @@ class AnthropicProvider(Provider):
"""
model = sanitize_anthropic_model_id(model)
# LiteLLM knows per-model cache and long-context rates, so let it price
- # the whole request rather than rebuilding the rate card here.
- cost = estimate_cost_from_tokens(
- model,
- input_tokens=input_tokens,
- output_tokens=output_tokens,
- cached_tokens=cached_tokens,
- now=now,
- )
- if cost is not None:
- return cost
+ # the whole request rather than rebuilding the rate card here -- unless
+ # its entry is missing the long-context rate this request needs.
+ if not _litellm_lacks_long_context_rate(model, input_tokens):
+ cost = estimate_cost_from_tokens(
+ model,
+ input_tokens=input_tokens,
+ output_tokens=output_tokens,
+ cached_tokens=cached_tokens,
+ now=now,
+ )
+ if cost is not None:
+ return cost
# Fall back to manual pricing
pricing = self._get_pricing(model)
diff --git a/tests/_pricing_models.py b/tests/_pricing_models.py
index 21dfa36f5..808f2720c 100644
--- a/tests/_pricing_models.py
+++ b/tests/_pricing_models.py
@@ -30,6 +30,10 @@ _CANDIDATES: tuple[str, ...] = (
"claude-sonnet-4-20250514",
"claude-opus-4-5-20251101",
"claude-opus-4-5",
+ # Last resort. By 2026-10, litellm had dropped the 200k-tier fields from
+ # every bare Anthropic id; only this Bedrock id still carries the full set,
+ # at the same rates as claude-sonnet-4-5.
+ "claude-sonnet-4-5-20250929-v1:0",
)
_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")
diff --git a/tests/test_counterfactual_pricing.py b/tests/test_counterfactual_pricing.py
index 9b8a10bdb..a373fd71f 100644
--- a/tests/test_counterfactual_pricing.py
+++ b/tests/test_counterfactual_pricing.py
@@ -172,12 +172,13 @@ def test_untagged_writes_default_to_the_five_minute_ttl():
def test_long_context_derives_the_one_hour_rate_and_says_so():
"""No catalog publishes a combined 1h + above-200k rate, so it derives."""
- rates = resolve_rates(SONNET, long_context=True)
+ model = anthropic_pricing_model("input_cost_per_token_above_200k_tokens")
+ rates = resolve_rates(model, long_context=True)
assert rates.basis == BASIS_CATALOG_TTL_RATIO
assert rates.write_1h / rates.uncached == pytest.approx(2.00, abs=1e-6)
# And it is the EXPENSIVE tier, not the base one.
- assert rates.uncached > resolve_rates(SONNET).uncached
+ assert rates.uncached > resolve_rates(model).uncached
# ── Provider agnosticism: every harness, every backend ────────────────────
diff --git a/tests/test_providers/test_anthropic.py b/tests/test_providers/test_anthropic.py
index 45c696f44..22ba814e1 100644
--- a/tests/test_providers/test_anthropic.py
+++ b/tests/test_providers/test_anthropic.py
@@ -136,6 +136,32 @@ class TestLongContextPricing:
)
assert cost == pytest.approx(expected, rel=1e-4)
+ def test_premium_survives_litellm_dropping_the_long_context_rate(self, provider, monkeypatch):
+ """A LiteLLM entry without the above-200K rate must not bill a long prompt at the base tier."""
+ litellm = pytest.importorskip("litellm")
+ import headroom.providers.anthropic as anthropic_module
+
+ # One map for both readers: the guard and litellm's own cost_per_token.
+ monkeypatch.setitem(
+ litellm.model_cost,
+ "claude-sonnet-4-5",
+ {
+ "litellm_provider": "anthropic",
+ "mode": "chat",
+ "input_cost_per_token": 3e-06,
+ "output_cost_per_token": 1.5e-05,
+ "cache_read_input_token_cost": 3e-07,
+ },
+ )
+
+ assert provider.estimate_cost(300_000, 5_000, "claude-sonnet-4-5", 0) == pytest.approx(
+ 1.9125, rel=1e-4
+ )
+ # Below the threshold the LiteLLM path still prices it.
+ assert (
+ anthropic_module._litellm_lacks_long_context_rate("claude-sonnet-4-5", 100_000) is False
+ )
+
def test_untiered_model_is_not_charged_a_premium(self, manual_provider):
# Opus is flat-rated across its whole window: 300K*$5 + 5K*$25 = $1.625.
cost = manual_provider.estimate_cost(300_000, 5_000, "claude-opus-4-5-20251101", 0)