The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from functools import lru_cache
from pathlib import Path
from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
FLEX_LONG_CONTEXT = {
"gpt-5.4": {
"input_cost_per_token_above_272k_tokens_flex": 2.5e-06,
"output_cost_per_token_above_272k_tokens_flex": 1.125e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07,
},
"gpt-5.4-pro": {
"input_cost_per_token_above_272k_tokens_flex": 3e-05,
"output_cost_per_token_above_272k_tokens_flex": 0.000135,
},
"gpt-5.5": {
"input_cost_per_token_above_272k_tokens_flex": 5e-06,
"output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
},
}
PRIORITY_LONG_CONTEXT = {
"gpt-5.6": {
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
},
"gpt-5.6-sol": {
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
},
"gpt-5.6-terra": {
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
},
"gpt-5.6-luna": {
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"output_cost_per_token_above_272k_tokens_priority": 3.6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
},
"gpt-6-astra": {
"input_cost_per_token_above_272k_tokens_priority": 4e-05,
"output_cost_per_token_above_272k_tokens_priority": 0.00015,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
},
"gpt-6-sol": {
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
"output_cost_per_token_above_272k_tokens_priority": 3e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
},
"gpt-6-luna": {
"input_cost_per_token_above_272k_tokens_priority": 4e-07,
"output_cost_per_token_above_272k_tokens_priority": 1.5e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
},
}
ULTRAFAST_LONG_CONTEXT = {
"gpt-6-astra": {
"input_cost_per_token_above_272k_tokens_ultrafast": 0.00012,
"output_cost_per_token_above_272k_tokens_ultrafast": 0.00045,
"cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-05,
"cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.00015,
}
}
EXPECTED: Final = {
model: {
**FLEX_LONG_CONTEXT.get(model, {}),
**PRIORITY_LONG_CONTEXT.get(model, {}),
**ULTRAFAST_LONG_CONTEXT.get(model, {}),
}
for model in {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT, **ULTRAFAST_LONG_CONTEXT}
}
NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5")
@pytest.fixture(autouse=True)
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.add_known_models()
@lru_cache(maxsize=2)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path) as f:
return json.load(f)
LONG_CONTEXT_PROMPT_TOKENS = 300_000
COMPLETION_TOKENS = 1_000
TIERED_COST_CASES = [
("gpt-5.4", "flex", 2.5e-06, 1.125e-05),
("gpt-5.4-pro", "flex", 3e-05, 0.000135),
("gpt-5.5", "flex", 5e-06, 2.25e-05),
("gpt-5.6", "priority", 1.6e-05, 6e-05),
("gpt-5.6-sol", "priority", 1.6e-05, 6e-05),
("gpt-5.6-terra", "priority", 8e-06, 3.6e-05),
("gpt-5.6-luna", "priority", 8e-07, 3.6e-06),
("gpt-6-astra", "priority", 4e-05, 0.00015),
("gpt-6-astra", "ultrafast", 0.00012, 0.00045),
("gpt-6-sol", "priority", 8e-06, 3e-05),
("gpt-6-luna", "priority", 4e-07, 1.5e-06),
]
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
def test_catalogs_contain_expected_tiered_long_context_rates(path: Path) -> None:
catalog: Final = _load(path)
assert {model: {key: catalog[model][key] for key in rates} for model, rates in EXPECTED.items()} == EXPECTED, (
"gpt-6-astra ultrafast rates per https://developers.openai.com/api/docs/pricing (2026-09-29)"
)
def test_get_model_info_preserves_expected_tiered_long_context_rates() -> None:
assert {
model: {key: litellm.get_model_info(model)[key] for key in rates} for model, rates in EXPECTED.items()
} == EXPECTED
@pytest.mark.parametrize(("model", "service_tier", "input_rate", "output_rate"), TIERED_COST_CASES)
def test_tiered_long_context_cost_uses_catalog_rates(
model: str, service_tier: str, input_rate: float, output_rate: float
) -> None:
usage: Final = Usage(
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
completion_tokens=COMPLETION_TOKENS,
total_tokens=LONG_CONTEXT_PROMPT_TOKENS + COMPLETION_TOKENS,
)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider="openai",
service_tier=service_tier,
)
assert prompt_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate)
assert completion_cost == pytest.approx(COMPLETION_TOKENS * output_rate)
def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None:
ultrafast_usage: Final = Usage(
prompt_tokens=300_000,
completion_tokens=1_000,
total_tokens=301_000,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200),
)
ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token(
model="gpt-6-astra",
usage=ultrafast_usage,
custom_llm_provider="openai",
service_tier="ultrafast",
)
standard_prompt_cost, standard_completion_cost = generic_cost_per_token(
model="gpt-6-astra",
usage=ultrafast_usage,
custom_llm_provider="openai",
)
below_threshold_usage: Final = Usage(
prompt_tokens=271_000,
completion_tokens=1_000,
total_tokens=272_000,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200),
)
below_threshold_prompt_cost, below_threshold_completion_cost = generic_cost_per_token(
model="gpt-6-astra",
usage=below_threshold_usage,
custom_llm_provider="openai",
service_tier="ultrafast",
)
assert (ultrafast_prompt_cost, ultrafast_completion_cost) == pytest.approx(
(299_700 * 0.00012 + 100 * 1.2e-05 + 200 * 0.00015, 1_000 * 0.00045)
)
assert ultrafast_prompt_cost + ultrafast_completion_cost == pytest.approx(36.4452)
assert (standard_prompt_cost, standard_completion_cost) == pytest.approx(
(299_700 * 0.00002 + 100 * 2e-06 + 200 * 2.5e-05, 1_000 * 7.5e-05)
)
assert (below_threshold_prompt_cost, below_threshold_completion_cost) == pytest.approx(
(270_700 * 6e-05 + 100 * 6e-06 + 200 * 7.5e-05, 1_000 * 0.0003)
)