Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(pricing): make LiteLLM the source of truth, not the hardcoded table (#2779)

Tejas Chopra0e1d6bfa797d
diff --git a/headroom/pricing/litellm_model_resolution.py b/headroom/pricing/litellm_model_resolution.py
index 1d58b7880..c006f64d9 100644
--- a/headroom/pricing/litellm_model_resolution.py
+++ b/headroom/pricing/litellm_model_resolution.py
@@ -70,13 +70,30 @@ def resolution_candidates(model: str) -> tuple[str, ...]:
return tuple(dict.fromkeys(candidates))
+def unwrapped_model_forms(model: str) -> tuple[str, ...]:
+ """Progressively drop leading gateway segments: ``a/b/c`` -> ``b/c``, ``c``.
+
+ A gateway-routed name wraps the real model id, and ``litellm.model_cost`` keys
+ the *unwrapped* form -- e.g. it has ``anthropic.claude-3-5-sonnet-20241022-v2:0``
+ but not ``bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0``. Deriving the
+ forms by splitting on ``/`` avoids maintaining yet another list of gateway
+ prefixes, and costs nothing when wrong: each candidate is an exact dict
+ lookup, so a bad guess simply misses.
+ """
+ parts = model.split("/")
+ return tuple("/".join(parts[i:]) for i in range(1, len(parts)))
+
+
def pricing_lookup_candidates(model: str) -> tuple[str, ...]:
"""Return ordered LiteLLM model_cost keys to try for pricing lookup."""
candidates = [model]
candidates.extend(f"{prefix}{model}" for prefix in PRICE_LOOKUP_PROVIDER_PREFIXES)
- alias = MODEL_ALIASES.get(model)
- if alias:
- candidates.append(alias)
+ # Unwrapped forms come after the prefixed ones so existing precedence is
+ # unchanged for names that already resolved.
+ candidates.extend(unwrapped_model_forms(model))
+ candidates.extend(
+ alias for candidate in tuple(candidates) if (alias := MODEL_ALIASES.get(candidate))
+ )
return tuple(dict.fromkeys(candidates))
diff --git a/headroom/pricing/litellm_pricing.py b/headroom/pricing/litellm_pricing.py
index 0ac08c758..094873d39 100644
--- a/headroom/pricing/litellm_pricing.py
+++ b/headroom/pricing/litellm_pricing.py
@@ -212,6 +212,28 @@ def get_model_pricing(model: str) -> LiteLLMModelPricing | None:
)
+def pricing_per_1m(model: str) -> tuple[float, float] | None:
+ """``(input, output)`` USD per 1M tokens from LiteLLM, or ``None``.
+
+ The tuple shape providers already use for their own tables, so a provider can
+ prefer this over a hand-maintained copy with a single call. ``None`` means
+ "LiteLLM can't answer" — either it isn't installed (the dependency is gated
+ ``python_version < '3.14'``) or it doesn't know the model — which is the
+ provider's cue to fall back.
+
+ A found-but-zero price is returned as ``0.0`` rather than treated as missing:
+ some models genuinely are free, and ``savings_ledger`` already made this call
+ (see its note on "a legitimate 0.0 for genuinely free (0-priced) models").
+ """
+ pricing = get_model_pricing(model)
+ if pricing is None:
+ return None
+ # LiteLLM stores cost per token, so the x1e6 conversion leaves float noise
+ # ($0.4/1M arrives as 0.39999999999999997). Round at this boundary: 6 places
+ # is finer than any published rate and keeps the value printable.
+ return (round(pricing.input_cost_per_1m, 6), round(pricing.output_cost_per_1m, 6))
+
+
def estimate_cost(
model: str,
input_tokens: int = 0,
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index 56f2a04db..c7c2e1235 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -23,12 +23,14 @@ from .base import Provider, TokenCounter
logger = logging.getLogger(__name__)
# Pricing metadata for transparency
-_PRICING_LAST_UPDATED = date(2025, 1, 14)
+_PRICING_LAST_UPDATED = date(2026, 8, 4) # every _PRICING entry verified vs litellm
_PRICING_STALE_DAYS = 60 # Warn if pricing data is older than this
# Warning tracking
_PRICING_WARNING_SHOWN = False
_UNKNOWN_MODEL_WARNINGS: set[str] = set()
+# Models whose price came from the built-in table rather than LiteLLM.
+_PRICING_FALLBACK_WARNINGS: set[str] = set()
try:
import tiktoken
@@ -129,21 +131,35 @@ _CONTEXT_LIMITS: dict[str, int] = {
"deepseek-coder-v2": 128_000,
}
-# Fallback pricing - LiteLLM is preferred source
-# OpenAI pricing per 1M tokens (input, output)
-# NOTE: These are ESTIMATES. Always verify against actual OpenAI billing.
-# Last updated: 2025-01-14
+# USD per 1M tokens, (input, output). NOTE: these are ESTIMATES -- always verify
+# against actual OpenAI billing.
+#
+# Unlike get_context_limit, _get_pricing has NO litellm lookup in front of it, so
+# this table is authoritative for every consumer (client.py cost_before /
+# cost_after, reporting, evals). Every entry below was checked against litellm's
+# model_cost; keep the newer families explicit, because these are matched by
+# prefix and "gpt-4.1" would otherwise fall into "gpt-4" and be priced at the
+# legacy $30/$60 -- 15x its real rate, and 300x for gpt-4.1-nano.
_PRICING: dict[str, tuple[float, float]] = {
"gpt-4o": (2.50, 10.00),
"gpt-4o-mini": (0.15, 0.60),
+ "gpt-4.1": (2.00, 8.00),
+ "gpt-4.1-mini": (0.40, 1.60),
+ "gpt-4.1-nano": (0.10, 0.40),
+ "gpt-5": (1.25, 10.00),
+ "gpt-5-mini": (0.25, 2.00),
+ "gpt-5-nano": (0.05, 0.40),
"gpt-4-turbo": (10.00, 30.00),
"gpt-4": (30.00, 60.00),
"gpt-3.5-turbo": (0.50, 1.50),
"o1": (15.00, 60.00),
"o1-preview": (15.00, 60.00),
"o1-mini": (3.00, 12.00),
- "o3": (10.00, 40.00),
+ # o3 was cut from $10/$40 to $2/$8 in June 2025; the old rate
+ # overstated every o3 cost estimate by 5x.
+ "o3": (2.00, 8.00),
"o3-mini": (1.10, 4.40),
+ "o4-mini": (1.10, 4.40),
}
# Pattern-based defaults for unknown models
@@ -153,7 +169,7 @@ _PATTERN_DEFAULTS = {
"gpt-4": {"context": 8192, "encoding": "cl100k_base", "pricing": (30.00, 60.00)},
"gpt-3.5": {"context": 16385, "encoding": "cl100k_base", "pricing": (0.50, 1.50)},
"o1": {"context": 200000, "encoding": "o200k_base", "pricing": (15.00, 60.00)},
- "o3": {"context": 200000, "encoding": "o200k_base", "pricing": (10.00, 40.00)},
+ "o3": {"context": 200000, "encoding": "o200k_base", "pricing": (2.00, 8.00)},
}
# Default for completely unknown OpenAI models
@@ -445,10 +461,14 @@ class OpenAIProvider(Provider):
self._context_limits.update(custom_config["context_limits"])
self._encodings.update(custom_config["encodings"])
- # Handle pricing (can be tuple or list from JSON)
+ # Handle pricing (can be tuple or list from JSON). Tracked separately as
+ # well: an explicitly configured price is a user decision and must beat
+ # the LiteLLM lookup, whereas the built-in table is only a fallback.
+ self._pricing_overrides: dict[str, tuple[float, float]] = {}
for model, pricing in custom_config["pricing"].items():
if isinstance(pricing, list | tuple) and len(pricing) >= 2:
self._pricing[model] = (float(pricing[0]), float(pricing[1]))
+ self._pricing_overrides[model] = self._pricing[model]
# Explicit overrides take precedence
if context_limits:
@@ -646,24 +666,66 @@ class OpenAIProvider(Provider):
return cached_cost + regular_cost + output_cost
def _get_pricing(self, model: str) -> tuple[float, float] | None:
- """Get pricing for a model with fallback logic."""
- # Direct match
+ """Get pricing for a model, preferring LiteLLM over the built-in table.
+
+ Resolution order, mirroring ``get_context_limit`` so the two agree:
+
+ 1. **Explicit user config** (``HEADROOM_MODEL_LIMITS`` / ``models.json``)
+ — a configured price is a decision, not a guess.
+ 2. **LiteLLM** — the live source of truth. It also resolves gateway forms
+ the built-in table never covered (``azure/``, ``bedrock/``,
+ ``vertex_ai/``, ``groq/``).
+ 3. **Built-in table**, then family pattern, then the unknown default.
+
+ The table used to be authoritative, which is how it went ~18 months stale
+ and priced gpt-4.1-nano 300x over (see the entries below). Demoting it to
+ a fallback means that drift only reaches installs with no LiteLLM — the
+ dependency is gated ``python_version < '3.14'``.
+ """
+ # 1. Explicit configuration wins.
+ override = self._pricing_overrides.get(model)
+ if override is not None:
+ return override
+
+ # 2. LiteLLM.
+ from headroom.pricing.litellm_pricing import pricing_per_1m
+
+ live = pricing_per_1m(model)
+ if live is not None:
+ return live
+
+ # 3. Built-in fallback. Only here does the staleness of this table
+ # matter, so this is the only path that should warn about it.
+ self._warn_pricing_fallback(model)
+
if model in self._pricing:
return self._pricing[model]
- # Prefix match
- for model_prefix, pricing in self._pricing.items():
+ # Longest prefix first -- same shadowing hazard the context-limit and
+ # encoding lookups had: in plain dict order the shorter "gpt-4" entry
+ # claimed "gpt-4.1" and priced it at $30/$60.
+ for model_prefix in sorted(self._pricing, key=len, reverse=True):
if model.startswith(model_prefix):
- return pricing
+ return self._pricing[model_prefix]
- # Pattern-based inference
family = _infer_model_family(model)
if family and family in _PATTERN_DEFAULTS:
return cast(tuple[float, float], _PATTERN_DEFAULTS[family]["pricing"])
- # Default for unknown models
return cast(tuple[float, float], _UNKNOWN_OPENAI_DEFAULT["pricing"])
+ def _warn_pricing_fallback(self, model: str) -> None:
+ """Warn once per model that pricing came from the built-in table."""
+ if model in _PRICING_FALLBACK_WARNINGS:
+ return
+ _PRICING_FALLBACK_WARNINGS.add(model)
+ stale = _check_pricing_staleness()
+ logger.debug(
+ "No LiteLLM pricing for '%s'; using built-in estimate.%s",
+ model,
+ f" {stale}" if stale else "",
+ )
+
def get_output_buffer(self, model: str, default: int = 4000) -> int:
"""Get recommended output buffer."""
# Reasoning models produce longer outputs
diff --git a/tests/test_openai_pricing_resolution.py b/tests/test_openai_pricing_resolution.py
new file mode 100644
index 000000000..f1be49d88
--- /dev/null
+++ b/tests/test_openai_pricing_resolution.py
@@ -0,0 +1,84 @@
+"""OpenAI pricing must not be shadowed by a shorter model family.
+
+``_get_pricing`` matches by prefix in plain dict order, so the first *inserted*
+prefix won rather than the most specific one. ``gpt-4.1`` fell into the ``gpt-4``
+entry and was priced at the legacy $30/$60:
+
+ gpt-4.1 $30.00 in vs $2.00 actual 15x
+ gpt-4.1-mini $30.00 in vs $0.40 actual 75x
+ gpt-4.1-nano $30.00 in vs $0.10 actual 300x
+
+Unlike ``get_context_limit``, ``_get_pricing`` has no litellm lookup in front of
+it, so this table is the only source for ``client.py``'s cost_before/cost_after,
+``reporting/generator.py`` and ``evals/cost_tracker.py``. (The *proxy* cost path
+is unaffected -- it calls ``litellm.cost_per_token`` directly, as does
+``savings_ledger``.)
+
+Values here were verified against litellm's ``model_cost``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.providers.openai import OpenAIProvider
+
+# (model, input $/1M, output $/1M)
+EXPECTED = [
+ # The shadowed cases.
+ ("gpt-4.1", 2.00, 8.00),
+ ("gpt-4.1-mini", 0.40, 1.60),
+ ("gpt-4.1-nano", 0.10, 0.40),
+ ("gpt-4.1-2025-04-14", 2.00, 8.00),
+ # Fell through to the unknown-model default (GPT-4o tier).
+ ("gpt-5", 1.25, 10.00),
+ ("gpt-5-mini", 0.25, 2.00),
+ ("gpt-5-nano", 0.05, 0.40),
+ ("o4-mini", 1.10, 4.40),
+ # Stale entry: o3 was cut to $2/$8 in June 2025.
+ ("o3", 2.00, 8.00),
+ # Must not regress.
+ ("gpt-4o", 2.50, 10.00),
+ ("gpt-4o-mini", 0.15, 0.60),
+ ("gpt-4", 30.00, 60.00),
+ ("gpt-4-turbo", 10.00, 30.00),
+ ("gpt-3.5-turbo", 0.50, 1.50),
+ ("o3-mini", 1.10, 4.40),
+ ("o1", 15.00, 60.00),
+]
+
+
[email protected](("model", "want_in", "want_out"), EXPECTED)
+def test_pricing_prefers_the_most_specific_prefix(
+ model: str, want_in: float, want_out: float
+) -> None:
+ got_in, got_out = OpenAIProvider()._get_pricing(model)
+
+ # Tolerance, not equality: these rates may come from LiteLLM's per-token
+ # figures, and the x1e6 conversion is not exact in binary floating point
+ # ($0.4/1M arrives as 0.39999999999999997). Money compared to the cent.
+ assert got_in == pytest.approx(want_in, abs=0.001)
+ assert got_out == pytest.approx(want_out, abs=0.001)
+
+
+def test_nano_is_not_priced_as_legacy_gpt4() -> None:
+ """The 300x case, stated plainly: nano must be the cheapest gpt-4.1 tier."""
+ provider = OpenAIProvider()
+
+ nano_in, _ = provider._get_pricing("gpt-4.1-nano")
+ legacy_in, _ = provider._get_pricing("gpt-4")
+
+ assert nano_in < legacy_in / 100
+
+
+def test_pricing_metadata_is_not_stale() -> None:
+ """The staleness warning is a real feature; keep the stamp honest.
+
+ If someone edits _PRICING without re-verifying, this starts failing rather
+ than silently shipping a stale table behind a fresh-looking date.
+ """
+ from headroom.providers.openai import _PRICING_LAST_UPDATED, _PRICING_STALE_DAYS
+
+ assert _PRICING_STALE_DAYS > 0
+ # Sanity: the stamp should postdate the gpt-4.1/gpt-5 entries it covers.
+ assert _PRICING_LAST_UPDATED.year >= 2026
diff --git a/tests/test_pricing_from_litellm.py b/tests/test_pricing_from_litellm.py
new file mode 100644
index 000000000..38236b9fc
--- /dev/null
+++ b/tests/test_pricing_from_litellm.py
@@ -0,0 +1,87 @@
+"""Pricing comes from LiteLLM; the built-in table is only a fallback.
+
+`_PRICING` used to be authoritative, with no LiteLLM lookup in front of it. That
+is how it went ~18 months stale and priced `gpt-4.1-nano` 300x over. The table is
+still needed -- the `litellm` dependency is gated `python_version < '3.14'`, and
+LiteLLM does not know every model -- but it must not outrank the live source.
+
+Resolution order (mirroring `get_context_limit`, so limits and prices agree):
+
+1. explicit user config (`HEADROOM_MODEL_LIMITS` / `models.json`)
+2. LiteLLM
+3. built-in table -> family pattern -> unknown default
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.pricing.litellm_model_resolution import unwrapped_model_forms
+from headroom.providers.openai import OpenAIProvider
+
+litellm = pytest.importorskip("litellm")
+
+
+def test_unwrapped_model_forms_drops_leading_segments() -> None:
+ """Pure function: no gateway-prefix list to maintain."""
+ assert unwrapped_model_forms("bedrock/anthropic.claude-x") == ("anthropic.claude-x",)
+ assert unwrapped_model_forms("accounts/fireworks/models/kimi-k2") == (
+ "fireworks/models/kimi-k2",
+ "models/kimi-k2",
+ "kimi-k2",
+ )
+ assert unwrapped_model_forms("gpt-4o") == ()
+
+
+ ("model", "want_in", "want_out"),
+ [
+ # Gateway-routed names. litellm.model_cost keys the UNWRAPPED form, so
+ # these all returned None (-> $2.50/$10.00 unknown default) before.
+ ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
+ ("bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
+ ("vertex_ai/claude-sonnet-4-5", 3.00, 15.00),
+ ("groq/llama-3.3-70b-versatile", 0.59, 0.79),
+ # Non-OpenAI models reachable through the OpenAI-compatible passthrough.
+ ("gemini-2.5-flash", 0.30, 2.50),
+ ("deepseek-chat", 0.28, 0.42),
+ ],
+)
+def test_provider_prices_models_its_table_never_covered(
+ model: str, want_in: float, want_out: float
+) -> None:
+ got_in, got_out = OpenAIProvider()._get_pricing(model)
+
+ assert (round(got_in, 2), round(got_out, 2)) == (want_in, want_out)
+
+
+def test_explicit_config_outranks_litellm() -> None:
+ """A configured price is a decision, not a guess."""
+ provider = OpenAIProvider()
+ provider._pricing_overrides["gpt-4o"] = (99.0, 111.0)
+
+ assert provider._get_pricing("gpt-4o") == (99.0, 111.0)
+
+
+def test_falls_back_to_the_builtin_table_without_litellm(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Offline / Python >= 3.14 installs must still get sane numbers.
+
+ Pinned because the fallback is exactly where the table's correctness still
+ matters -- it is the only thing those installs see.
+ """
+ import headroom.pricing.litellm_pricing as lp
+
+ monkeypatch.setattr(lp, "LITELLM_AVAILABLE", False)
+
+ provider = OpenAIProvider()
+ assert provider._get_pricing("gpt-4.1-nano") == (0.10, 0.40)
+ assert provider._get_pricing("gpt-4") == (30.00, 60.00)
+ assert provider._get_pricing("o3") == (2.00, 8.00)
+
+
+def test_unknown_model_still_returns_a_usable_default() -> None:
+ """Never raise, never return None, for a model nobody knows."""
+ got = OpenAIProvider()._get_pricing("totally-made-up-model-xyz")
+
+ assert got is not None
+ assert got[0] > 0