Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tests): ask litellm for a model it prices instead of naming a retired one (#3722)
Tejas Chopra0ad5e68ca559
diff --git a/dashboard-cache-ttl-main.png b/dashboard-cache-ttl-main.pngindex 3433b1c56..19a69cd20 100644Binary files a/dashboard-cache-ttl-main.png and b/dashboard-cache-ttl-main.png differdiff --git a/headroom/pricing/litellm_model_resolution.py b/headroom/pricing/litellm_model_resolution.pyindex 0a9242001..f06586980 100644--- a/headroom/pricing/litellm_model_resolution.py+++ b/headroom/pricing/litellm_model_resolution.py@@ -27,15 +27,25 @@ class LiteLLMModelPrefixRule:# Aliases for models removed from LiteLLM's cost database (retired/renamed).# Maps old model name -> current LiteLLM key that has equivalent pricing.+# Retired model ids, aliased to a CURRENTLY PRICED model of the same tier.+#+# An alias target must itself still be in litellm's table. These pointed at+# claude-sonnet-4-20250514 until litellm pruned it on 2026-09-23, at which point+# all three aliases resolved to nothing and every one of these models fell+# through to the unknown-model default (the GPT-4o tier, $2.50/$10.00) -- the+# same class of silent mispricing the Haiku note below warns about. The guard+# test fails the build if a target is pruned again, so this cannot rot quietly.+#+# The undated target is deliberate: it survives a dated release being retired.MODEL_ALIASES: dict[str, str] = {# Claude 3.5 Sonnet retired Feb 2026, pricing same as claude-sonnet-4-20250514- "claude-3-5-sonnet-20241022": "claude-sonnet-4-20250514",- "claude-3-5-sonnet-20240620": "claude-sonnet-4-20250514",+ "claude-3-5-sonnet-20241022": "claude-sonnet-4-5",+ "claude-3-5-sonnet-20240620": "claude-sonnet-4-5",# Claude 3 Sonnet retired. It was a Sonnet-tier model ($3/$15 per 1M# in/out) — same price as claude-sonnet-4-20250514 — so alias it there.# The old target, claude-3-haiku-20240307 ($0.25/$1.25), is a different# (Haiku) tier and underpriced every cost/savings figure ~12x.- "claude-3-sonnet-20240229": "claude-sonnet-4-20250514",+ "claude-3-sonnet-20240229": "claude-sonnet-4-5",}diff --git a/tests/_pricing_models.py b/tests/_pricing_models.pynew file mode 100644index 000000000..21dfa36f5--- /dev/null+++ b/tests/_pricing_models.py@@ -0,0 +1,64 @@+"""Pick a model litellm actually prices, instead of hardcoding one it may retire.++``litellm.model_cost`` is downloaded from GitHub at import time, so it is live+third-party data. BerriAI prunes retired models from it: on 2026-09-23+``claude-sonnet-4-20250514`` disappeared and every test that priced it began+failing with ``KeyError: 'input_cost_per_token'`` — on every open pull request+at once, with no change on our side.++The model id in those tests is incidental. They assert that Headroom's cost+arithmetic agrees with litellm's numbers, not that any particular model is+priced correctly, so the fix is to stop naming a specific release and instead+ask for *a* model carrying the fields the test needs.++Pinning to the copy of the table vendored in the litellm wheel is not the+alternative it looks like: the two maps are complementary, not ordered. The+vendored map keeps retired ids but predates current models+(``claude-sonnet-5``), and its older entries lack newer fields such as+``input_cost_per_token_above_200k_tokens`` entirely.+"""++from __future__ import annotations++import pytest++#: Preference order, newest first. A test takes the first entry that carries+#: every field it needs, so retiring one is a no-op until the list runs dry.+_CANDIDATES: tuple[str, ...] = (+ "claude-sonnet-4-5-20250929",+ "claude-sonnet-4-5",+ "claude-sonnet-4-20250514",+ "claude-opus-4-5-20251101",+ "claude-opus-4-5",+)++_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")+++def anthropic_pricing_model(*required_fields: str) -> str:+ """Return a currently-priced Anthropic model carrying ``required_fields``.++ Always includes the base input/output costs. Raises with an actionable+ message rather than skipping: if litellm prices none of these, the pricing+ tests are not measuring anything and that should be loud.+ """+ import litellm++ needed = set(_BASE_FIELDS) | set(required_fields)+ for model in _CANDIDATES:+ info = litellm.model_cost.get(model)+ if isinstance(info, dict) and needed <= set(info):+ return model++ raise AssertionError(+ "litellm prices none of the candidate models with the fields "+ f"{sorted(needed)}. It most likely retired them from "+ "model_prices_and_context_window.json; add a current model id to "+ "_CANDIDATES in tests/_pricing_models.py (newest first)."+ )++[email protected](scope="session")+def anthropic_model() -> str:+ """Session fixture wrapper for tests that prefer injection over a constant."""+ return anthropic_pricing_model()diff --git a/tests/test_cost_budget_basis.py b/tests/test_cost_budget_basis.pyindex cdb7ad1d6..e0b93480f 100644--- a/tests/test_cost_budget_basis.py+++ b/tests/test_cost_budget_basis.py@@ -22,13 +22,15 @@ from tests._dotenv import (importorskip_no_env_leak,load_env_overrides,)+from tests._pricing_models import anthropic_pricing_model_env_overrides = load_env_overrides()apply_dotenv = autouse_apply_env(_env_overrides)importorskip_no_env_leak("litellm")-MODEL = "claude-sonnet-4-20250514"++MODEL = anthropic_pricing_model()@pytest.fixture(autouse=True)diff --git a/tests/test_cost_card_attribution.py b/tests/test_cost_card_attribution.pyindex eb50f0ce3..271973555 100644--- a/tests/test_cost_card_attribution.py+++ b/tests/test_cost_card_attribution.py@@ -16,13 +16,19 @@ from tests._dotenv import (importorskip_no_env_leak,load_env_overrides,)+from tests._pricing_models import anthropic_pricing_model_env_overrides = load_env_overrides()apply_dotenv = autouse_apply_env(_env_overrides)importorskip_no_env_leak("litellm")-MODEL = "claude-sonnet-4-20250514"++MODEL = anthropic_pricing_model(+ "input_cost_per_token_above_200k_tokens",+ "cache_creation_input_token_cost",+ "cache_read_input_token_cost",+)def _prices(model: str = MODEL) -> tuple[float, float, float]:diff --git a/tests/test_cost_tracker_counterfactual.py b/tests/test_cost_tracker_counterfactual.pyindex 63cc123cd..9e12d090e 100644--- a/tests/test_cost_tracker_counterfactual.py+++ b/tests/test_cost_tracker_counterfactual.py@@ -11,6 +11,7 @@ from tests._dotenv import (importorskip_no_env_leak,load_env_overrides,)+from tests._pricing_models import anthropic_pricing_model_env_overrides = load_env_overrides()apply_dotenv = autouse_apply_env(_env_overrides)@@ -18,12 +19,15 @@ apply_dotenv = autouse_apply_env(_env_overrides)importorskip_no_env_leak("litellm")+MODEL = anthropic_pricing_model()++def test_savings_at_list_price():"""savings_usd = tokens_saved * model list input price."""from headroom.proxy.server import CostTrackerct = CostTracker()- model = "claude-sonnet-4-20250514"+ model = MODELct.record_tokens(model,@@ -54,7 +58,7 @@ def test_savings_monotonic():from headroom.proxy.server import CostTrackerct = CostTracker()- model = "claude-sonnet-4-20250514"+ model = MODELct.record_tokens(model, tokens_saved=10_000, tokens_sent=5_000)stats1 = ct.stats()@@ -71,7 +75,7 @@ def test_savings_zero_when_no_tokens_saved():from headroom.proxy.server import CostTrackerct = CostTracker()- model = "claude-sonnet-4-20250514"+ model = MODELct.record_tokens(model, tokens_saved=0, tokens_sent=5_000)stats = ct.stats()@@ -99,7 +103,7 @@ def test_multi_model_savings():ct = CostTracker()- ct.record_tokens("claude-sonnet-4-20250514", tokens_saved=50_000, tokens_sent=10_000)+ ct.record_tokens(MODEL, tokens_saved=50_000, tokens_sent=10_000)ct.record_tokens("claude-haiku-4-5-20251001", tokens_saved=50_000, tokens_sent=10_000)stats = ct.stats()@@ -117,7 +121,7 @@ def test_no_cost_without_headroom_field():from headroom.proxy.server import CostTrackerct = CostTracker()- ct.record_tokens("claude-sonnet-4-20250514", tokens_saved=10_000, tokens_sent=5_000)+ ct.record_tokens(MODEL, tokens_saved=10_000, tokens_sent=5_000)stats = ct.stats()assert "cost_without_headroom_usd" not in stats@@ -137,7 +141,7 @@ def test_budget_enforced_after_recording_costs():# ~$1.50+ of Sonnet input at list price — far over the budgetct.record_tokens(- "claude-sonnet-4-20250514",+ MODEL,tokens_saved=0,tokens_sent=500_000,uncached_tokens=500_000,@@ -158,7 +162,7 @@ def test_budget_input_cost_counted_without_usage_breakdown():ct = CostTracker(budget_limit_usd=100.0)ct.record_tokens(- "claude-sonnet-4-20250514",+ MODEL,tokens_saved=0,tokens_sent=500_000,)diff --git a/tests/test_counterfactual_pricing.py b/tests/test_counterfactual_pricing.pyindex ae395452c..9b8a10bdb 100644--- a/tests/test_counterfactual_pricing.py+++ b/tests/test_counterfactual_pricing.py@@ -20,6 +20,7 @@ from tests._dotenv import (importorskip_no_env_leak,load_env_overrides,)+from tests._pricing_models import anthropic_pricing_model_env_overrides = load_env_overrides()apply_dotenv = autouse_apply_env(_env_overrides)@@ -40,7 +41,7 @@ from headroom.pricing.counterfactual import ( # noqa: E402weakest_basis,)-SONNET = "claude-sonnet-4-20250514"+SONNET = anthropic_pricing_model()def _warm_anthropic() -> CacheMix:diff --git a/tests/test_model_alias_targets_are_priced.py b/tests/test_model_alias_targets_are_priced.pynew file mode 100644index 000000000..26884dfef--- /dev/null+++ b/tests/test_model_alias_targets_are_priced.py@@ -0,0 +1,62 @@+"""An alias target that litellm no longer prices is worse than no alias.++`MODEL_ALIASES` exists so a retired model id still prices at its own tier.+Every target pointed at `claude-sonnet-4-20250514` until litellm pruned it on+2026-09-23; all three aliases then resolved to nothing and those models fell+through to the unknown-model default -- the GPT-4o tier -- which silently+misprices them. Nothing failed, because no test asserted the targets were live.+"""++from __future__ import annotations++import pytest++from tests._dotenv import importorskip_no_env_leak++importorskip_no_env_leak("litellm")++from headroom.pricing.litellm_model_resolution import MODEL_ALIASES # noqa: E402+from headroom.pricing.litellm_pricing import get_model_pricing # noqa: E402++[email protected](("retired", "target"), sorted(MODEL_ALIASES.items()))+def test_model_aliases_point_at_something_litellm_still_prices(retired: str, target: str) -> None:+ """The target must be priced, or the alias is a silent no-op."""+ assert get_model_pricing(target) is not None, (+ f"MODEL_ALIASES[{retired!r}] points at {target!r}, which litellm no longer "+ "prices. Repoint it at a current model of the same tier "+ "(headroom/pricing/litellm_model_resolution.py)."+ )++[email protected](("retired", "target"), sorted(MODEL_ALIASES.items()))+def test_a_retired_id_prices_at_its_alias_rate(retired: str, target: str) -> None:+ """The point of the alias: the retired id resolves to the target's numbers."""+ priced = get_model_pricing(retired)+ assert priced is not None, f"{retired!r} did not resolve through its alias"++ target_priced = get_model_pricing(target)+ assert target_priced is not None+ assert (priced.input_cost_per_1m, priced.output_cost_per_1m) == (+ target_priced.input_cost_per_1m,+ target_priced.output_cost_per_1m,+ )+++def test_retired_sonnets_are_not_priced_at_the_unknown_default() -> None:+ """Regression for the failure mode this whole file is about.++ The unknown-model fallback is the GPT-4o tier. A Sonnet-tier model landing+ there is wrong in both directions and is exactly what happened when the+ alias targets went stale.+ """+ gpt4o = get_model_pricing("gpt-4o")+ assert gpt4o is not None++ for retired in MODEL_ALIASES:+ priced = get_model_pricing(retired)+ assert priced is not None+ assert (priced.input_cost_per_1m, priced.output_cost_per_1m) != (+ gpt4o.input_cost_per_1m,+ gpt4o.output_cost_per_1m,+ ), f"{retired!r} is priced at the GPT-4o tier — its alias is not resolving"diff --git a/tests/test_models.py b/tests/test_models.pyindex 69b3e0c85..117d7c031 100644--- a/tests/test_models.py+++ b/tests/test_models.py@@ -13,6 +13,7 @@ from headroom.models import (list_models,register_model,)+from tests._pricing_models import anthropic_pricing_modelclass TestModelInfo:@@ -285,8 +286,9 @@ class TestBuiltInModels:info = get_model_info("claude-3-5-sonnet-20241022")assert info.provider == "anthropic"assert info.context_window == 200000- # Pricing fetched from LiteLLM (falls back to alias for retired models)- pricing = ModelRegistry.get_pricing("claude-sonnet-4-20250514")+ # Pricing comes from litellm's live table, so name a model it currently+ # prices; the retired-id path is the MODEL_ALIASES assertion below.+ pricing = ModelRegistry.get_pricing(anthropic_pricing_model())assert pricing is not Noneassert pricing[0] == 3.00 # input cost per 1Massert pricing[1] == 15.00 # output cost per 1Mdiff --git a/tests/test_pricing_from_litellm.py b/tests/test_pricing_from_litellm.pyindex 38236b9fc..d9cd7b8a9 100644--- a/tests/test_pricing_from_litellm.py+++ b/tests/test_pricing_from_litellm.py@@ -41,7 +41,12 @@ def test_unwrapped_model_forms_drops_leading_segments() -> None:("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),("bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),("vertex_ai/claude-sonnet-4-5", 3.00, 15.00),- ("groq/llama-3.3-70b-versatile", 0.59, 0.79),+ # Any gateway-prefixed name here must be one litellm still prices:+ # when it prunes a model the unwrap finds nothing and _get_pricing+ # silently returns the $2.50/$10.00 GPT-4o default, which is what+ # this test exists to catch. litellm dropped+ # groq/llama-3.3-70b-versatile on 2026-09-23; see issue #3732.+ ("groq/llama-guard-3-8b", 0.20, 0.20),# Non-OpenAI models reachable through the OpenAI-compatible passthrough.("gemini-2.5-flash", 0.30, 2.50),("deepseek-chat", 0.28, 0.42),diff --git a/tests/test_pricing_litellm.py b/tests/test_pricing_litellm.pyindex c088f06ac..63b1a4c42 100644--- a/tests/test_pricing_litellm.py+++ b/tests/test_pricing_litellm.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom types import SimpleNamespacefrom headroom.pricing import litellm_pricing+from headroom.pricing.litellm_model_resolution import MODEL_ALIASESdef test_litellm_helpers_when_dependency_is_unavailable(monkeypatch) -> None:@@ -72,9 +73,14 @@ def test_litellm_model_pricing_uses_provider_prefixes(monkeypatch) -> None:def test_litellm_model_pricing_uses_aliases_and_zero_cost_defaults(monkeypatch) -> None:+ # Keyed off the alias table rather than restating its target: this named+ # "claude-sonnet-4-20250514" and broke when the alias moved, even though the+ # behaviour under test (alias resolution + None costs reading as zero) was+ # unchanged.+ alias_target = MODEL_ALIASES["claude-3-5-sonnet-20241022"]fake_litellm = SimpleNamespace(model_cost={- "claude-sonnet-4-20250514": {+ alias_target: {"input_cost_per_token": None,"output_cost_per_token": None,}diff --git a/tests/test_pricing_litellm_model_resolution.py b/tests/test_pricing_litellm_model_resolution.pyindex dcfb21485..b89965790 100644--- a/tests/test_pricing_litellm_model_resolution.py+++ b/tests/test_pricing_litellm_model_resolution.py@@ -47,9 +47,15 @@ def test_retired_claude_3_sonnet_aliases_to_sonnet_tier_not_haiku() -> None:alias = MODEL_ALIASES["claude-3-sonnet-20240229"]assert "haiku" not in alias+ assert "sonnet" in alias# Same-tier target as the other retired-Sonnet aliases.assert alias == MODEL_ALIASES["claude-3-5-sonnet-20241022"]- assert alias == "claude-sonnet-4-20250514"+ # Deliberately not a specific release: this asserted+ # "claude-sonnet-4-20250514" until litellm pruned that id, at which point+ # the alias had to move and this test broke for naming the target rather+ # than the property it cares about. That the target is *priced*, and priced+ # at Sonnet rates rather than the unknown-model default, is asserted in+ # tests/test_model_alias_targets_are_priced.py.def test_resolve_litellm_model_name_returns_first_known_candidate() -> None:diff --git a/tests/test_savings_ledger.py b/tests/test_savings_ledger.pyindex 1c7ec7b6a..e51c7ef33 100644--- a/tests/test_savings_ledger.py+++ b/tests/test_savings_ledger.py@@ -8,6 +8,9 @@ from datetime import datetime, timedelta, timezoneimport pytestfrom headroom import savings_ledger as L+from tests._pricing_models import anthropic_pricing_model++MODEL = anthropic_pricing_model()UTC = timezone.utc@@ -417,7 +420,7 @@ def test_v1_events_still_aggregate_alongside_v2(tmp_path):"after": 6_000,"saved": 4_000,"cost_usd": 0.012,- "model": "claude-sonnet-4-20250514",+ "model": MODEL,"client": "claude-code","source": "proxy",}@@ -427,7 +430,7 @@ def test_v1_events_still_aggregate_alongside_v2(tmp_path):sl.record_savings_event(tokens_before=108_000,tokens_after=100_000,- model="claude-sonnet-4-20250514",+ model=MODEL,client="claude-code",source="proxy",saved_compression=1_000,diff --git a/tests/test_savings_provider_matrix.py b/tests/test_savings_provider_matrix.pyindex c6755d804..55d350342 100644--- a/tests/test_savings_provider_matrix.py+++ b/tests/test_savings_provider_matrix.py@@ -25,6 +25,13 @@ from tests._dotenv import (importorskip_no_env_leak,load_env_overrides,)+from tests._pricing_models import anthropic_pricing_model++MODEL = anthropic_pricing_model(+ "input_cost_per_token_above_200k_tokens",+ "cache_creation_input_token_cost",+ "cache_read_input_token_cost",+)_env_overrides = load_env_overrides()apply_dotenv = autouse_apply_env(_env_overrides)@@ -36,14 +43,14 @@ from headroom.proxy.savings_tracker import estimate_request_savings_usd # noqa:# One warm turn per provider, in that provider's own reporting shape.WARM_TURNS = {"anthropic": {- "model": "claude-sonnet-4-20250514",+ "model": MODEL,"cache_read_tokens": 180_000,"cache_write_tokens": 2_000,"cache_write_5m_tokens": 2_000,"uncached_input_tokens": 500,},"bedrock": {- "model": "claude-sonnet-4-20250514",+ "model": MODEL,"cache_read_tokens": 180_000,"cache_write_tokens": 2_000,"cache_write_5m_tokens": 2_000,@@ -132,7 +139,7 @@ def test_a_provider_that_reports_no_cache_data_prices_at_list_and_says_so():not presented as a measurement."""priced = estimate_request_savings_usd(- model="claude-sonnet-4-20250514",+ model=MODEL,compression_tokens_saved=5_000,tool_schema_tokens_saved=5_000,)@@ -180,7 +187,7 @@ def test_the_headline_degrades_to_the_weakest_provider_in_a_mixed_session():not present its total as catalog-grade."""priced = estimate_request_savings_usd(- model="claude-sonnet-4-20250514",+ model=MODEL,# Priced against a real mix...compression_tokens_saved=5_000,cache_read_tokens=100_000,@@ -193,7 +200,7 @@ def test_the_headline_degrades_to_the_weakest_provider_in_a_mixed_session():# ...but the same layers with nothing reported cannot claim the same.blind = estimate_request_savings_usd(- model="claude-sonnet-4-20250514",+ model=MODEL,compression_tokens_saved=5_000,)assert blind["basis"] == "no-mix"@@ -206,7 +213,7 @@ def test_long_context_requests_price_at_the_above_200k_tier():request's savings at base rates understates them by half."""base = estimate_request_savings_usd(- model="claude-sonnet-4-20250514",+ model=MODEL,compression_tokens_saved=5_000,cache_write_tokens=50_000,cache_write_5m_tokens=50_000,@@ -214,7 +221,7 @@ def test_long_context_requests_price_at_the_above_200k_tier():provider="anthropic",)long_ctx = estimate_request_savings_usd(- model="claude-sonnet-4-20250514",+ model=MODEL,compression_tokens_saved=5_000,cache_write_tokens=250_000,cache_write_5m_tokens=250_000,