Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(pricing): keep the long-context premium when litellm drops the 200k rate (#4109)
Tejas Chopra8c1beb20aeb8
diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.pyindex 7634f3ccc..e8b29eed4 100644--- a/headroom/providers/anthropic.py+++ b/headroom/providers/anthropic.py@@ -30,7 +30,8 @@ from headroom.pricing.deepseek_tiers import (OFF_PEAK_RATES_PER_1M,off_peak_rates,)-from headroom.pricing.litellm_pricing import estimate_cost_from_tokens+from headroom.pricing.litellm_model_resolution import pricing_lookup_candidates+from headroom.pricing.litellm_pricing import estimate_cost_from_tokens, get_litellm_model_costfrom headroom.tokenizers.base import (TokenCountCache,coerce_countable_text,@@ -219,8 +220,9 @@ def _apply_long_context_premium() -> dict[str, float]:"""Return ``pricing`` scaled by the long-context premium where it applies.- Used only on the manual fallback path; the LiteLLM path already applies the- published above-threshold rates itself.+ Used on the manual path, which also serves long-context requests whose+ LiteLLM entry has no above-threshold rate (see+ ``_litellm_lacks_long_context_rate``)."""if input_tokens <= _LONG_CONTEXT_THRESHOLD:return pricing@@ -229,6 +231,26 @@ def _apply_long_context_premium(return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}+def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:+ """True when this request needs the long-context premium but LiteLLM has no rate for it.++ LiteLLM's live price map has dropped ``input_cost_per_token_above_200k_tokens``+ from the bare Anthropic ids before, and pricing such a request through it then+ bills the base tier: about half the real cost of a long session. The manual+ table still carries the premium, so the caller uses it instead.+ """+ if input_tokens <= _LONG_CONTEXT_THRESHOLD:+ return False+ if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):+ return False+ cost_data = get_litellm_model_cost()+ for candidate in pricing_lookup_candidates(model):+ info = cost_data.get(candidate)+ if isinstance(info, dict):+ return "input_cost_per_token_above_200k_tokens" not in info+ return False # LiteLLM doesn't know the model; it returns None and we fall back anyway++# Default limits for pattern-based inference# Used when a model isn't in the explicit list but matches a known pattern_PATTERN_DEFAULTS = {@@ -837,16 +859,18 @@ class AnthropicProvider(Provider):"""model = sanitize_anthropic_model_id(model)# LiteLLM knows per-model cache and long-context rates, so let it price- # the whole request rather than rebuilding the rate card here.- cost = estimate_cost_from_tokens(- model,- input_tokens=input_tokens,- output_tokens=output_tokens,- cached_tokens=cached_tokens,- now=now,- )- if cost is not None:- return cost+ # the whole request rather than rebuilding the rate card here -- unless+ # its entry is missing the long-context rate this request needs.+ if not _litellm_lacks_long_context_rate(model, input_tokens):+ cost = estimate_cost_from_tokens(+ model,+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cached_tokens=cached_tokens,+ now=now,+ )+ if cost is not None:+ return cost# Fall back to manual pricingpricing = self._get_pricing(model)diff --git a/tests/_pricing_models.py b/tests/_pricing_models.pyindex 21dfa36f5..808f2720c 100644--- a/tests/_pricing_models.py+++ b/tests/_pricing_models.py@@ -30,6 +30,10 @@ _CANDIDATES: tuple[str, ...] = ("claude-sonnet-4-20250514","claude-opus-4-5-20251101","claude-opus-4-5",+ # Last resort. By 2026-10, litellm had dropped the 200k-tier fields from+ # every bare Anthropic id; only this Bedrock id still carries the full set,+ # at the same rates as claude-sonnet-4-5.+ "claude-sonnet-4-5-20250929-v1:0",)_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")diff --git a/tests/test_counterfactual_pricing.py b/tests/test_counterfactual_pricing.pyindex 9b8a10bdb..a373fd71f 100644--- a/tests/test_counterfactual_pricing.py+++ b/tests/test_counterfactual_pricing.py@@ -172,12 +172,13 @@ def test_untagged_writes_default_to_the_five_minute_ttl():def test_long_context_derives_the_one_hour_rate_and_says_so():"""No catalog publishes a combined 1h + above-200k rate, so it derives."""- rates = resolve_rates(SONNET, long_context=True)+ model = anthropic_pricing_model("input_cost_per_token_above_200k_tokens")+ rates = resolve_rates(model, long_context=True)assert rates.basis == BASIS_CATALOG_TTL_RATIOassert rates.write_1h / rates.uncached == pytest.approx(2.00, abs=1e-6)# And it is the EXPENSIVE tier, not the base one.- assert rates.uncached > resolve_rates(SONNET).uncached+ assert rates.uncached > resolve_rates(model).uncached# ── Provider agnosticism: every harness, every backend ────────────────────diff --git a/tests/test_providers/test_anthropic.py b/tests/test_providers/test_anthropic.pyindex 45c696f44..22ba814e1 100644--- a/tests/test_providers/test_anthropic.py+++ b/tests/test_providers/test_anthropic.py@@ -136,6 +136,32 @@ class TestLongContextPricing:)assert cost == pytest.approx(expected, rel=1e-4)+ def test_premium_survives_litellm_dropping_the_long_context_rate(self, provider, monkeypatch):+ """A LiteLLM entry without the above-200K rate must not bill a long prompt at the base tier."""+ litellm = pytest.importorskip("litellm")+ import headroom.providers.anthropic as anthropic_module++ # One map for both readers: the guard and litellm's own cost_per_token.+ monkeypatch.setitem(+ litellm.model_cost,+ "claude-sonnet-4-5",+ {+ "litellm_provider": "anthropic",+ "mode": "chat",+ "input_cost_per_token": 3e-06,+ "output_cost_per_token": 1.5e-05,+ "cache_read_input_token_cost": 3e-07,+ },+ )++ assert provider.estimate_cost(300_000, 5_000, "claude-sonnet-4-5", 0) == pytest.approx(+ 1.9125, rel=1e-4+ )+ # Below the threshold the LiteLLM path still prices it.+ assert (+ anthropic_module._litellm_lacks_long_context_rate("claude-sonnet-4-5", 100_000) is False+ )+def test_untiered_model_is_not_charged_a_premium(self, manual_provider):# Opus is flat-rated across its whole window: 300K*$5 + 5K*$25 = $1.625.cost = manual_provider.estimate_cost(300_000, 5_000, "claude-opus-4-5-20251101", 0)