Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tests): ask litellm for a model it prices instead of naming a retired one (#3722)

Tejas Chopra0ad5e68ca559
diff --git a/dashboard-cache-ttl-main.png b/dashboard-cache-ttl-main.png
index 3433b1c56..19a69cd20 100644
Binary files a/dashboard-cache-ttl-main.png and b/dashboard-cache-ttl-main.png differ
diff --git a/headroom/pricing/litellm_model_resolution.py b/headroom/pricing/litellm_model_resolution.py
index 0a9242001..f06586980 100644
--- a/headroom/pricing/litellm_model_resolution.py
+++ b/headroom/pricing/litellm_model_resolution.py
@@ -27,15 +27,25 @@ class LiteLLMModelPrefixRule:
# Aliases for models removed from LiteLLM's cost database (retired/renamed).
# Maps old model name -> current LiteLLM key that has equivalent pricing.
+# Retired model ids, aliased to a CURRENTLY PRICED model of the same tier.
+#
+# An alias target must itself still be in litellm's table. These pointed at
+# claude-sonnet-4-20250514 until litellm pruned it on 2026-09-23, at which point
+# all three aliases resolved to nothing and every one of these models fell
+# through to the unknown-model default (the GPT-4o tier, $2.50/$10.00) -- the
+# same class of silent mispricing the Haiku note below warns about. The guard
+# test fails the build if a target is pruned again, so this cannot rot quietly.
+#
+# The undated target is deliberate: it survives a dated release being retired.
MODEL_ALIASES: dict[str, str] = {
# Claude 3.5 Sonnet retired Feb 2026, pricing same as claude-sonnet-4-20250514
- "claude-3-5-sonnet-20241022": "claude-sonnet-4-20250514",
- "claude-3-5-sonnet-20240620": "claude-sonnet-4-20250514",
+ "claude-3-5-sonnet-20241022": "claude-sonnet-4-5",
+ "claude-3-5-sonnet-20240620": "claude-sonnet-4-5",
# Claude 3 Sonnet retired. It was a Sonnet-tier model ($3/$15 per 1M
# in/out) — same price as claude-sonnet-4-20250514 — so alias it there.
# The old target, claude-3-haiku-20240307 ($0.25/$1.25), is a different
# (Haiku) tier and underpriced every cost/savings figure ~12x.
- "claude-3-sonnet-20240229": "claude-sonnet-4-20250514",
+ "claude-3-sonnet-20240229": "claude-sonnet-4-5",
}
diff --git a/tests/_pricing_models.py b/tests/_pricing_models.py
new file mode 100644
index 000000000..21dfa36f5
--- /dev/null
+++ b/tests/_pricing_models.py
@@ -0,0 +1,64 @@
+"""Pick a model litellm actually prices, instead of hardcoding one it may retire.
+
+``litellm.model_cost`` is downloaded from GitHub at import time, so it is live
+third-party data. BerriAI prunes retired models from it: on 2026-09-23
+``claude-sonnet-4-20250514`` disappeared and every test that priced it began
+failing with ``KeyError: 'input_cost_per_token'`` — on every open pull request
+at once, with no change on our side.
+
+The model id in those tests is incidental. They assert that Headroom's cost
+arithmetic agrees with litellm's numbers, not that any particular model is
+priced correctly, so the fix is to stop naming a specific release and instead
+ask for *a* model carrying the fields the test needs.
+
+Pinning to the copy of the table vendored in the litellm wheel is not the
+alternative it looks like: the two maps are complementary, not ordered. The
+vendored map keeps retired ids but predates current models
+(``claude-sonnet-5``), and its older entries lack newer fields such as
+``input_cost_per_token_above_200k_tokens`` entirely.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+#: Preference order, newest first. A test takes the first entry that carries
+#: every field it needs, so retiring one is a no-op until the list runs dry.
+_CANDIDATES: tuple[str, ...] = (
+ "claude-sonnet-4-5-20250929",
+ "claude-sonnet-4-5",
+ "claude-sonnet-4-20250514",
+ "claude-opus-4-5-20251101",
+ "claude-opus-4-5",
+)
+
+_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")
+
+
+def anthropic_pricing_model(*required_fields: str) -> str:
+ """Return a currently-priced Anthropic model carrying ``required_fields``.
+
+ Always includes the base input/output costs. Raises with an actionable
+ message rather than skipping: if litellm prices none of these, the pricing
+ tests are not measuring anything and that should be loud.
+ """
+ import litellm
+
+ needed = set(_BASE_FIELDS) | set(required_fields)
+ for model in _CANDIDATES:
+ info = litellm.model_cost.get(model)
+ if isinstance(info, dict) and needed <= set(info):
+ return model
+
+ raise AssertionError(
+ "litellm prices none of the candidate models with the fields "
+ f"{sorted(needed)}. It most likely retired them from "
+ "model_prices_and_context_window.json; add a current model id to "
+ "_CANDIDATES in tests/_pricing_models.py (newest first)."
+ )
+
+
[email protected](scope="session")
+def anthropic_model() -> str:
+ """Session fixture wrapper for tests that prefer injection over a constant."""
+ return anthropic_pricing_model()
diff --git a/tests/test_cost_budget_basis.py b/tests/test_cost_budget_basis.py
index cdb7ad1d6..e0b93480f 100644
--- a/tests/test_cost_budget_basis.py
+++ b/tests/test_cost_budget_basis.py
@@ -22,13 +22,15 @@ from tests._dotenv import (
importorskip_no_env_leak,
load_env_overrides,
)
+from tests._pricing_models import anthropic_pricing_model
_env_overrides = load_env_overrides()
apply_dotenv = autouse_apply_env(_env_overrides)
importorskip_no_env_leak("litellm")
-MODEL = "claude-sonnet-4-20250514"
+
+MODEL = anthropic_pricing_model()
@pytest.fixture(autouse=True)
diff --git a/tests/test_cost_card_attribution.py b/tests/test_cost_card_attribution.py
index eb50f0ce3..271973555 100644
--- a/tests/test_cost_card_attribution.py
+++ b/tests/test_cost_card_attribution.py
@@ -16,13 +16,19 @@ from tests._dotenv import (
importorskip_no_env_leak,
load_env_overrides,
)
+from tests._pricing_models import anthropic_pricing_model
_env_overrides = load_env_overrides()
apply_dotenv = autouse_apply_env(_env_overrides)
importorskip_no_env_leak("litellm")
-MODEL = "claude-sonnet-4-20250514"
+
+MODEL = anthropic_pricing_model(
+ "input_cost_per_token_above_200k_tokens",
+ "cache_creation_input_token_cost",
+ "cache_read_input_token_cost",
+)
def _prices(model: str = MODEL) -> tuple[float, float, float]:
diff --git a/tests/test_cost_tracker_counterfactual.py b/tests/test_cost_tracker_counterfactual.py
index 63cc123cd..9e12d090e 100644
--- a/tests/test_cost_tracker_counterfactual.py
+++ b/tests/test_cost_tracker_counterfactual.py
@@ -11,6 +11,7 @@ from tests._dotenv import (
importorskip_no_env_leak,
load_env_overrides,
)
+from tests._pricing_models import anthropic_pricing_model
_env_overrides = load_env_overrides()
apply_dotenv = autouse_apply_env(_env_overrides)
@@ -18,12 +19,15 @@ apply_dotenv = autouse_apply_env(_env_overrides)
importorskip_no_env_leak("litellm")
+MODEL = anthropic_pricing_model()
+
+
def test_savings_at_list_price():
"""savings_usd = tokens_saved * model list input price."""
from headroom.proxy.server import CostTracker
ct = CostTracker()
- model = "claude-sonnet-4-20250514"
+ model = MODEL
ct.record_tokens(
model,
@@ -54,7 +58,7 @@ def test_savings_monotonic():
from headroom.proxy.server import CostTracker
ct = CostTracker()
- model = "claude-sonnet-4-20250514"
+ model = MODEL
ct.record_tokens(model, tokens_saved=10_000, tokens_sent=5_000)
stats1 = ct.stats()
@@ -71,7 +75,7 @@ def test_savings_zero_when_no_tokens_saved():
from headroom.proxy.server import CostTracker
ct = CostTracker()
- model = "claude-sonnet-4-20250514"
+ model = MODEL
ct.record_tokens(model, tokens_saved=0, tokens_sent=5_000)
stats = ct.stats()
@@ -99,7 +103,7 @@ def test_multi_model_savings():
ct = CostTracker()
- ct.record_tokens("claude-sonnet-4-20250514", tokens_saved=50_000, tokens_sent=10_000)
+ ct.record_tokens(MODEL, tokens_saved=50_000, tokens_sent=10_000)
ct.record_tokens("claude-haiku-4-5-20251001", tokens_saved=50_000, tokens_sent=10_000)
stats = ct.stats()
@@ -117,7 +121,7 @@ def test_no_cost_without_headroom_field():
from headroom.proxy.server import CostTracker
ct = CostTracker()
- ct.record_tokens("claude-sonnet-4-20250514", tokens_saved=10_000, tokens_sent=5_000)
+ ct.record_tokens(MODEL, tokens_saved=10_000, tokens_sent=5_000)
stats = ct.stats()
assert "cost_without_headroom_usd" not in stats
@@ -137,7 +141,7 @@ def test_budget_enforced_after_recording_costs():
# ~$1.50+ of Sonnet input at list price — far over the budget
ct.record_tokens(
- "claude-sonnet-4-20250514",
+ MODEL,
tokens_saved=0,
tokens_sent=500_000,
uncached_tokens=500_000,
@@ -158,7 +162,7 @@ def test_budget_input_cost_counted_without_usage_breakdown():
ct = CostTracker(budget_limit_usd=100.0)
ct.record_tokens(
- "claude-sonnet-4-20250514",
+ MODEL,
tokens_saved=0,
tokens_sent=500_000,
)
diff --git a/tests/test_counterfactual_pricing.py b/tests/test_counterfactual_pricing.py
index ae395452c..9b8a10bdb 100644
--- a/tests/test_counterfactual_pricing.py
+++ b/tests/test_counterfactual_pricing.py
@@ -20,6 +20,7 @@ from tests._dotenv import (
importorskip_no_env_leak,
load_env_overrides,
)
+from tests._pricing_models import anthropic_pricing_model
_env_overrides = load_env_overrides()
apply_dotenv = autouse_apply_env(_env_overrides)
@@ -40,7 +41,7 @@ from headroom.pricing.counterfactual import ( # noqa: E402
weakest_basis,
)
-SONNET = "claude-sonnet-4-20250514"
+SONNET = anthropic_pricing_model()
def _warm_anthropic() -> CacheMix:
diff --git a/tests/test_model_alias_targets_are_priced.py b/tests/test_model_alias_targets_are_priced.py
new file mode 100644
index 000000000..26884dfef
--- /dev/null
+++ b/tests/test_model_alias_targets_are_priced.py
@@ -0,0 +1,62 @@
+"""An alias target that litellm no longer prices is worse than no alias.
+
+`MODEL_ALIASES` exists so a retired model id still prices at its own tier.
+Every target pointed at `claude-sonnet-4-20250514` until litellm pruned it on
+2026-09-23; all three aliases then resolved to nothing and those models fell
+through to the unknown-model default -- the GPT-4o tier -- which silently
+misprices them. Nothing failed, because no test asserted the targets were live.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+from headroom.pricing.litellm_model_resolution import MODEL_ALIASES # noqa: E402
+from headroom.pricing.litellm_pricing import get_model_pricing # noqa: E402
+
+
[email protected](("retired", "target"), sorted(MODEL_ALIASES.items()))
+def test_model_aliases_point_at_something_litellm_still_prices(retired: str, target: str) -> None:
+ """The target must be priced, or the alias is a silent no-op."""
+ assert get_model_pricing(target) is not None, (
+ f"MODEL_ALIASES[{retired!r}] points at {target!r}, which litellm no longer "
+ "prices. Repoint it at a current model of the same tier "
+ "(headroom/pricing/litellm_model_resolution.py)."
+ )
+
+
[email protected](("retired", "target"), sorted(MODEL_ALIASES.items()))
+def test_a_retired_id_prices_at_its_alias_rate(retired: str, target: str) -> None:
+ """The point of the alias: the retired id resolves to the target's numbers."""
+ priced = get_model_pricing(retired)
+ assert priced is not None, f"{retired!r} did not resolve through its alias"
+
+ target_priced = get_model_pricing(target)
+ assert target_priced is not None
+ assert (priced.input_cost_per_1m, priced.output_cost_per_1m) == (
+ target_priced.input_cost_per_1m,
+ target_priced.output_cost_per_1m,
+ )
+
+
+def test_retired_sonnets_are_not_priced_at_the_unknown_default() -> None:
+ """Regression for the failure mode this whole file is about.
+
+ The unknown-model fallback is the GPT-4o tier. A Sonnet-tier model landing
+ there is wrong in both directions and is exactly what happened when the
+ alias targets went stale.
+ """
+ gpt4o = get_model_pricing("gpt-4o")
+ assert gpt4o is not None
+
+ for retired in MODEL_ALIASES:
+ priced = get_model_pricing(retired)
+ assert priced is not None
+ assert (priced.input_cost_per_1m, priced.output_cost_per_1m) != (
+ gpt4o.input_cost_per_1m,
+ gpt4o.output_cost_per_1m,
+ ), f"{retired!r} is priced at the GPT-4o tier — its alias is not resolving"
diff --git a/tests/test_models.py b/tests/test_models.py
index 69b3e0c85..117d7c031 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -13,6 +13,7 @@ from headroom.models import (
list_models,
register_model,
)
+from tests._pricing_models import anthropic_pricing_model
class TestModelInfo:
@@ -285,8 +286,9 @@ class TestBuiltInModels:
info = get_model_info("claude-3-5-sonnet-20241022")
assert info.provider == "anthropic"
assert info.context_window == 200000
- # Pricing fetched from LiteLLM (falls back to alias for retired models)
- pricing = ModelRegistry.get_pricing("claude-sonnet-4-20250514")
+ # Pricing comes from litellm's live table, so name a model it currently
+ # prices; the retired-id path is the MODEL_ALIASES assertion below.
+ pricing = ModelRegistry.get_pricing(anthropic_pricing_model())
assert pricing is not None
assert pricing[0] == 3.00 # input cost per 1M
assert pricing[1] == 15.00 # output cost per 1M
diff --git a/tests/test_pricing_from_litellm.py b/tests/test_pricing_from_litellm.py
index 38236b9fc..d9cd7b8a9 100644
--- a/tests/test_pricing_from_litellm.py
+++ b/tests/test_pricing_from_litellm.py
@@ -41,7 +41,12 @@ def test_unwrapped_model_forms_drops_leading_segments() -> None:
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
("bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
("vertex_ai/claude-sonnet-4-5", 3.00, 15.00),
- ("groq/llama-3.3-70b-versatile", 0.59, 0.79),
+ # Any gateway-prefixed name here must be one litellm still prices:
+ # when it prunes a model the unwrap finds nothing and _get_pricing
+ # silently returns the $2.50/$10.00 GPT-4o default, which is what
+ # this test exists to catch. litellm dropped
+ # groq/llama-3.3-70b-versatile on 2026-09-23; see issue #3732.
+ ("groq/llama-guard-3-8b", 0.20, 0.20),
# Non-OpenAI models reachable through the OpenAI-compatible passthrough.
("gemini-2.5-flash", 0.30, 2.50),
("deepseek-chat", 0.28, 0.42),
diff --git a/tests/test_pricing_litellm.py b/tests/test_pricing_litellm.py
index c088f06ac..63b1a4c42 100644
--- a/tests/test_pricing_litellm.py
+++ b/tests/test_pricing_litellm.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from types import SimpleNamespace
from headroom.pricing import litellm_pricing
+from headroom.pricing.litellm_model_resolution import MODEL_ALIASES
def test_litellm_helpers_when_dependency_is_unavailable(monkeypatch) -> None:
@@ -72,9 +73,14 @@ def test_litellm_model_pricing_uses_provider_prefixes(monkeypatch) -> None:
def test_litellm_model_pricing_uses_aliases_and_zero_cost_defaults(monkeypatch) -> None:
+ # Keyed off the alias table rather than restating its target: this named
+ # "claude-sonnet-4-20250514" and broke when the alias moved, even though the
+ # behaviour under test (alias resolution + None costs reading as zero) was
+ # unchanged.
+ alias_target = MODEL_ALIASES["claude-3-5-sonnet-20241022"]
fake_litellm = SimpleNamespace(
model_cost={
- "claude-sonnet-4-20250514": {
+ alias_target: {
"input_cost_per_token": None,
"output_cost_per_token": None,
}
diff --git a/tests/test_pricing_litellm_model_resolution.py b/tests/test_pricing_litellm_model_resolution.py
index dcfb21485..b89965790 100644
--- a/tests/test_pricing_litellm_model_resolution.py
+++ b/tests/test_pricing_litellm_model_resolution.py
@@ -47,9 +47,15 @@ def test_retired_claude_3_sonnet_aliases_to_sonnet_tier_not_haiku() -> None:
alias = MODEL_ALIASES["claude-3-sonnet-20240229"]
assert "haiku" not in alias
+ assert "sonnet" in alias
# Same-tier target as the other retired-Sonnet aliases.
assert alias == MODEL_ALIASES["claude-3-5-sonnet-20241022"]
- assert alias == "claude-sonnet-4-20250514"
+ # Deliberately not a specific release: this asserted
+ # "claude-sonnet-4-20250514" until litellm pruned that id, at which point
+ # the alias had to move and this test broke for naming the target rather
+ # than the property it cares about. That the target is *priced*, and priced
+ # at Sonnet rates rather than the unknown-model default, is asserted in
+ # tests/test_model_alias_targets_are_priced.py.
def test_resolve_litellm_model_name_returns_first_known_candidate() -> None:
diff --git a/tests/test_savings_ledger.py b/tests/test_savings_ledger.py
index 1c7ec7b6a..e51c7ef33 100644
--- a/tests/test_savings_ledger.py
+++ b/tests/test_savings_ledger.py
@@ -8,6 +8,9 @@ from datetime import datetime, timedelta, timezone
import pytest
from headroom import savings_ledger as L
+from tests._pricing_models import anthropic_pricing_model
+
+MODEL = anthropic_pricing_model()
UTC = timezone.utc
@@ -417,7 +420,7 @@ def test_v1_events_still_aggregate_alongside_v2(tmp_path):
"after": 6_000,
"saved": 4_000,
"cost_usd": 0.012,
- "model": "claude-sonnet-4-20250514",
+ "model": MODEL,
"client": "claude-code",
"source": "proxy",
}
@@ -427,7 +430,7 @@ def test_v1_events_still_aggregate_alongside_v2(tmp_path):
sl.record_savings_event(
tokens_before=108_000,
tokens_after=100_000,
- model="claude-sonnet-4-20250514",
+ model=MODEL,
client="claude-code",
source="proxy",
saved_compression=1_000,
diff --git a/tests/test_savings_provider_matrix.py b/tests/test_savings_provider_matrix.py
index c6755d804..55d350342 100644
--- a/tests/test_savings_provider_matrix.py
+++ b/tests/test_savings_provider_matrix.py
@@ -25,6 +25,13 @@ from tests._dotenv import (
importorskip_no_env_leak,
load_env_overrides,
)
+from tests._pricing_models import anthropic_pricing_model
+
+MODEL = anthropic_pricing_model(
+ "input_cost_per_token_above_200k_tokens",
+ "cache_creation_input_token_cost",
+ "cache_read_input_token_cost",
+)
_env_overrides = load_env_overrides()
apply_dotenv = autouse_apply_env(_env_overrides)
@@ -36,14 +43,14 @@ from headroom.proxy.savings_tracker import estimate_request_savings_usd # noqa:
# One warm turn per provider, in that provider's own reporting shape.
WARM_TURNS = {
"anthropic": {
- "model": "claude-sonnet-4-20250514",
+ "model": MODEL,
"cache_read_tokens": 180_000,
"cache_write_tokens": 2_000,
"cache_write_5m_tokens": 2_000,
"uncached_input_tokens": 500,
},
"bedrock": {
- "model": "claude-sonnet-4-20250514",
+ "model": MODEL,
"cache_read_tokens": 180_000,
"cache_write_tokens": 2_000,
"cache_write_5m_tokens": 2_000,
@@ -132,7 +139,7 @@ def test_a_provider_that_reports_no_cache_data_prices_at_list_and_says_so():
not presented as a measurement.
"""
priced = estimate_request_savings_usd(
- model="claude-sonnet-4-20250514",
+ model=MODEL,
compression_tokens_saved=5_000,
tool_schema_tokens_saved=5_000,
)
@@ -180,7 +187,7 @@ def test_the_headline_degrades_to_the_weakest_provider_in_a_mixed_session():
not present its total as catalog-grade.
"""
priced = estimate_request_savings_usd(
- model="claude-sonnet-4-20250514",
+ model=MODEL,
# Priced against a real mix...
compression_tokens_saved=5_000,
cache_read_tokens=100_000,
@@ -193,7 +200,7 @@ def test_the_headline_degrades_to_the_weakest_provider_in_a_mixed_session():
# ...but the same layers with nothing reported cannot claim the same.
blind = estimate_request_savings_usd(
- model="claude-sonnet-4-20250514",
+ model=MODEL,
compression_tokens_saved=5_000,
)
assert blind["basis"] == "no-mix"
@@ -206,7 +213,7 @@ def test_long_context_requests_price_at_the_above_200k_tier():
request's savings at base rates understates them by half.
"""
base = estimate_request_savings_usd(
- model="claude-sonnet-4-20250514",
+ model=MODEL,
compression_tokens_saved=5_000,
cache_write_tokens=50_000,
cache_write_5m_tokens=50_000,
@@ -214,7 +221,7 @@ def test_long_context_requests_price_at_the_above_200k_tier():
provider="anthropic",
)
long_ctx = estimate_request_savings_usd(
- model="claude-sonnet-4-20250514",
+ model=MODEL,
compression_tokens_saved=5_000,
cache_write_tokens=250_000,
cache_write_5m_tokens=250_000,