Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: price cached tokens across SDKs and Manage Models (#1390)

Aman Agarwal815bde6bd447
diff --git a/assets/pricing.json b/assets/pricing.json
index 692cecf3..d2088b18 100644
--- a/assets/pricing.json
+++ b/assets/pricing.json
@@ -1,16 +1,16 @@
{
"embeddings": {
"text-embedding-ada-002": 0.0001,
- "text-embedding-3-small": 0.00002,
+ "text-embedding-3-small": 2e-05,
"text-embedding-3-large": 0.00013,
"ada": 0.0001,
- "ada-v2": 0.00010,
+ "ada-v2": 0.0001,
"text-ada-001": 0.0001,
"azure_text-embedding-ada-002": 0.0001,
- "azure_text-embedding-3-small": 0.00002,
+ "azure_text-embedding-3-small": 2e-05,
"azure_text-embedding-3-large": 0.00013,
"azure_ada": 0.0001,
- "azure_ada-v2": 0.00010,
+ "azure_ada-v2": 0.0001,
"azure_text-ada-001": 0.0001,
"embed-english-v3.0": 0.0001,
"embed-multilingual-v3.0": 0.0001,
@@ -21,7 +21,7 @@
"embed-multilingual-v2.0": 0.0001,
"mistral-embed": 0.0001,
"amazon.titan-embed-text-v1": 0.0001,
- "amazon.titan-embed-text-v2": 0.00002,
+ "amazon.titan-embed-text-v2": 2e-05,
"textembedding-gecko": 0.0001,
"textembedding-gecko@001": 0.0001,
"textembedding-gecko@002": 0.0001,
@@ -30,31 +30,31 @@
"images": {
"dall-e-3": {
"standard": {
- "1024x1024": 0.040,
- "1024x1792": 0.020,
- "1792x1024": 0.080
+ "1024x1024": 0.04,
+ "1024x1792": 0.02,
+ "1792x1024": 0.08
},
"hd": {
- "1024x1024": 0.080,
- "1024x1792": 0.120,
- "1792x1024": 0.120
+ "1024x1024": 0.08,
+ "1024x1792": 0.12,
+ "1792x1024": 0.12
}
},
"azure_dall-e-3": {
"standard": {
- "1024x1024": 0.040,
- "1024x1792": 0.020,
- "1792x1024": 0.080
+ "1024x1024": 0.04,
+ "1024x1792": 0.02,
+ "1792x1024": 0.08
},
"hd": {
- "1024x1024": 0.080,
- "1024x1792": 0.120,
- "1792x1024": 0.120
+ "1024x1024": 0.08,
+ "1024x1792": 0.12,
+ "1792x1024": 0.12
}
},
"dall-e-2": {
"standard": {
- "1024x1024": 0.020,
+ "1024x1024": 0.02,
"512x512": 0.018,
"256x256": 0.016
}
@@ -112,8 +112,8 @@
}
},
"audio": {
- "tts-1" : 0.015,
- "tts-1-hd": 0.030,
+ "tts-1": 0.015,
+ "tts-1-hd": 0.03,
"eleven_multilingual_v2": 0.24,
"eleven_multilingual_v1": 0.24,
"eleven_monolingual_v1": 0.24,
@@ -122,11 +122,11 @@
"eleven_english_sts_v2": 0.24,
"eleven_multilingual_sts_v2": 0.24,
"best": 0.00010277777,
- "nano": 0.00003333333,
- "whisper-large-v3": 0.00003083333,
- "whisper-large-v3-turbo": 0.00001111111,
+ "nano": 3.333333e-05,
+ "whisper-large-v3": 3.083333e-05,
+ "whisper-large-v3-turbo": 1.111111e-05,
"canopylabs/orpheus-v1-english": 0.022,
- "canopylabs/orpheus-arabic-saudi": 0.040
+ "canopylabs/orpheus-arabic-saudi": 0.04
},
"chat": {
"gpt-4.1-nano-2025-04-14": {
@@ -135,31 +135,38 @@
},
"gpt-4.1-nano": {
"promptPrice": 0.0001,
- "completionPrice": 0.0004
+ "completionPrice": 0.0004,
+ "cacheReadPrice": 2.5e-05
},
"gpt-4.1-mini-2025-04-14": {
"promptPrice": 0.0004,
- "completionPrice": 0.0016
+ "completionPrice": 0.0016,
+ "cacheReadPrice": 0.0001
},
"gpt-4.1-mini": {
"promptPrice": 0.0004,
- "completionPrice": 0.0016
+ "completionPrice": 0.0016,
+ "cacheReadPrice": 0.0001
},
"gpt-4.1-2025-04-14": {
"promptPrice": 0.002,
- "completionPrice": 0.008
+ "completionPrice": 0.008,
+ "cacheReadPrice": 0.0005
},
"gpt-4.1": {
"promptPrice": 0.002,
- "completionPrice": 0.008
+ "completionPrice": 0.008,
+ "cacheReadPrice": 0.0005
},
"gpt-4o": {
- "promptPrice": 0.0005,
- "completionPrice": 0.0015
+ "promptPrice": 0.0025,
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.00125
},
"gpt-4o-2024-08-06": {
"promptPrice": 0.0025,
- "completionPrice": 0.010
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.00125
},
"gpt-4o-2024-05-13": {
"promptPrice": 0.005,
@@ -167,27 +174,33 @@
},
"gpt-4o-mini": {
"promptPrice": 0.00015,
- "completionPrice": 0.0006
+ "completionPrice": 0.0006,
+ "cacheReadPrice": 7.5e-05
},
"gpt-4o-mini-2024-07-18": {
"promptPrice": 0.00015,
- "completionPrice": 0.0006
+ "completionPrice": 0.0006,
+ "cacheReadPrice": 7.5e-05
},
"o1-mini": {
- "promptPrice": 0.003,
- "completionPrice": 0.012
+ "promptPrice": 0.0011,
+ "completionPrice": 0.0044,
+ "cacheReadPrice": 0.00055
},
"o1-mini-2024-09-12": {
- "promptPrice": 0.003,
- "completionPrice": 0.012
+ "promptPrice": 0.0011,
+ "completionPrice": 0.0044,
+ "cacheReadPrice": 0.00055
},
"o1-preview": {
"promptPrice": 0.015,
- "completionPrice": 0.06
+ "completionPrice": 0.06,
+ "cacheReadPrice": 0.0075
},
"o1-preview-2024-09-12": {
"promptPrice": 0.015,
- "completionPrice": 0.06
+ "completionPrice": 0.06,
+ "cacheReadPrice": 0.0075
},
"gpt-3.5-turbo": {
"promptPrice": 0.0005,
@@ -207,7 +220,7 @@
},
"azure_gpt-35-turbo-instruct": {
"promptPrice": 0.0015,
- "completionPrice": 0.0020
+ "completionPrice": 0.002
},
"gpt-4": {
"promptPrice": 0.03,
@@ -264,7 +277,7 @@
"claude-3-haiku-20240307": {
"promptPrice": 0.00025,
"completionPrice": 0.00125,
- "cacheReadPrice": 0.00003,
+ "cacheReadPrice": 3e-05,
"cacheCreationPrice": 0.0003
},
"claude-3-5-sonnet-20240620": {
@@ -282,7 +295,7 @@
"claude-3-5-haiku-20241022": {
"promptPrice": 0.0008,
"completionPrice": 0.004,
- "cacheReadPrice": 0.00008,
+ "cacheReadPrice": 8e-05,
"cacheCreationPrice": 0.001
},
"claude-3-7-sonnet-20250219": {
@@ -383,19 +396,23 @@
},
"gpt-5": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125
},
"gpt-5-codex": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125
},
"gpt-5-mini": {
"promptPrice": 0.00025,
- "completionPrice": 0.002
+ "completionPrice": 0.002,
+ "cacheReadPrice": 2.5e-05
},
"gpt-5-nano": {
- "promptPrice": 0.00005,
- "completionPrice": 0.0004
+ "promptPrice": 5e-05,
+ "completionPrice": 0.0004,
+ "cacheReadPrice": 5e-06
},
"gpt-5-pro": {
"promptPrice": 0.015,
@@ -403,23 +420,28 @@
},
"gpt-5.1": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125
},
"gpt-5.1-codex": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125
},
"gpt-5.1-codex-max": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125
},
"gpt-5.2": {
"promptPrice": 0.00175,
- "completionPrice": 0.014
+ "completionPrice": 0.014,
+ "cacheReadPrice": 0.000175
},
"gpt-5.2-codex": {
"promptPrice": 0.00175,
- "completionPrice": 0.014
+ "completionPrice": 0.014,
+ "cacheReadPrice": 0.000175
},
"gpt-5.2-pro": {
"promptPrice": 0.021,
@@ -427,19 +449,23 @@
},
"gpt-5.3-codex": {
"promptPrice": 0.00175,
- "completionPrice": 0.014
+ "completionPrice": 0.014,
+ "cacheReadPrice": 0.000175
},
"gpt-5.4": {
"promptPrice": 0.0025,
- "completionPrice": 0.015
+ "completionPrice": 0.015,
+ "cacheReadPrice": 0.00025
},
"gpt-5.4-mini": {
"promptPrice": 0.00075,
- "completionPrice": 0.0045
+ "completionPrice": 0.0045,
+ "cacheReadPrice": 7.5e-05
},
"gpt-5.4-nano": {
"promptPrice": 0.0002,
- "completionPrice": 0.00125
+ "completionPrice": 0.00125,
+ "cacheReadPrice": 2e-05
},
"gpt-5.4-pro": {
"promptPrice": 0.03,
@@ -447,7 +473,8 @@
},
"gpt-5.5": {
"promptPrice": 0.005,
- "completionPrice": 0.03
+ "completionPrice": 0.03,
+ "cacheReadPrice": 0.0005
},
"gpt-5.5-pro": {
"promptPrice": 0.03,
@@ -455,15 +482,18 @@
},
"o3": {
"promptPrice": 0.002,
- "completionPrice": 0.008
+ "completionPrice": 0.008,
+ "cacheReadPrice": 0.0005
},
"o3-mini": {
"promptPrice": 0.0011,
- "completionPrice": 0.0044
+ "completionPrice": 0.0044,
+ "cacheReadPrice": 0.00055
},
"o4-mini": {
"promptPrice": 0.0011,
- "completionPrice": 0.0044
+ "completionPrice": 0.0044,
+ "cacheReadPrice": 0.000275
},
"command": {
"promptPrice": 0.001,
@@ -555,7 +585,7 @@
},
"cohere.command-text-v14": {
"promptPrice": 0.0015,
- "completionPrice": 0.0020
+ "completionPrice": 0.002
},
"cohere.command-light-text-v14": {
"promptPrice": 0.0003,
@@ -574,12 +604,16 @@
"completionPrice": 0.0015
},
"gemini-1.5-flash": {
- "promptPrice": 0.000075,
- "completionPrice": 0.0003
+ "promptPrice": 7.5e-05,
+ "completionPrice": 0.0003,
+ "cacheReadPrice": 1.875e-05,
+ "cacheCreationPrice": 7.5e-05
},
"gemini-1.5-pro": {
"promptPrice": 0.00125,
- "completionPrice": 0.005
+ "completionPrice": 0.005,
+ "cacheReadPrice": 0.0003125,
+ "cacheCreationPrice": 0.00125
},
"gemini-1.0-pro-002": {
"promptPrice": 0.0005,
@@ -591,63 +625,93 @@
},
"gemini-1.5-pro-preview-0409": {
"promptPrice": 0.005,
- "completionPrice": 0.015
+ "completionPrice": 0.015,
+ "cacheReadPrice": 0.00125,
+ "cacheCreationPrice": 0.005
},
"gemini-1.5-pro-preview-0514": {
"promptPrice": 0.005,
- "completionPrice": 0.015
+ "completionPrice": 0.015,
+ "cacheReadPrice": 0.00125,
+ "cacheCreationPrice": 0.005
},
"gemini-1.5-flash-preview-0514": {
"promptPrice": 0.0005,
- "completionPrice": 0.0015
+ "completionPrice": 0.0015,
+ "cacheReadPrice": 0.000125,
+ "cacheCreationPrice": 0.0005
},
"gemini-2.0-flash": {
"promptPrice": 0.0001,
- "completionPrice": 0.0004
+ "completionPrice": 0.0004,
+ "cacheReadPrice": 1e-05,
+ "cacheCreationPrice": 0.0001
},
"gemini-2.0-flash-lite": {
- "promptPrice": 0.000075,
- "completionPrice": 0.0003
+ "promptPrice": 7.5e-05,
+ "completionPrice": 0.0003,
+ "cacheReadPrice": 7.5e-06,
+ "cacheCreationPrice": 7.5e-05
},
"gemini-2.5-flash": {
"promptPrice": 0.0003,
- "completionPrice": 0.0025
+ "completionPrice": 0.0025,
+ "cacheReadPrice": 3e-05,
+ "cacheCreationPrice": 0.0003
},
"gemini-2.5-flash-preview": {
"promptPrice": 0.0003,
- "completionPrice": 0.0025
+ "completionPrice": 0.0025,
+ "cacheReadPrice": 3e-05,
+ "cacheCreationPrice": 0.0003
},
"gemini-2.5-flash-lite": {
"promptPrice": 0.0001,
- "completionPrice": 0.0004
+ "completionPrice": 0.0004,
+ "cacheReadPrice": 1e-05,
+ "cacheCreationPrice": 0.0001
},
"gemini-2.5-flash-lite-preview": {
"promptPrice": 0.0001,
- "completionPrice": 0.0004
+ "completionPrice": 0.0004,
+ "cacheReadPrice": 1e-05,
+ "cacheCreationPrice": 0.0001
},
"gemini-2.5-pro": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125,
+ "cacheCreationPrice": 0.00125
},
"gemini-2.5-pro-preview": {
"promptPrice": 0.00125,
- "completionPrice": 0.01
+ "completionPrice": 0.01,
+ "cacheReadPrice": 0.000125,
+ "cacheCreationPrice": 0.00125
},
"gemini-3-flash": {
"promptPrice": 0.0005,
- "completionPrice": 0.003
+ "completionPrice": 0.003,
+ "cacheReadPrice": 5e-05,
+ "cacheCreationPrice": 0.0005
},
"gemini-3.1-flash-lite": {
"promptPrice": 0.00025,
- "completionPrice": 0.0015
+ "completionPrice": 0.0015,
+ "cacheReadPrice": 2.5e-05,
+ "cacheCreationPrice": 0.00025
},
"gemini-3.1-pro": {
"promptPrice": 0.002,
- "completionPrice": 0.012
+ "completionPrice": 0.012,
+ "cacheReadPrice": 0.0002,
+ "cacheCreationPrice": 0.002
},
"gemini-3.1-pro-preview": {
"promptPrice": 0.002,
- "completionPrice": 0.012
+ "completionPrice": 0.012,
+ "cacheReadPrice": 0.0002,
+ "cacheCreationPrice": 0.002
},
"text-bison": {
"promptPrice": 0.001,
@@ -690,7 +754,7 @@
"completionPrice": 0.002
},
"llama3-8b-8192": {
- "promptPrice": 0.00005,
+ "promptPrice": 5e-05,
"completionPrice": 0.0001
},
"llama3-70b-8192": {
@@ -734,8 +798,8 @@
"completionPrice": 0.0001
},
"reka-spark": {
- "promptPrice": 0.00005,
- "completionPrice": 0.00005
+ "promptPrice": 5e-05,
+ "completionPrice": 5e-05
},
"grok-beta": {
"promptPrice": 0.0005,
@@ -755,11 +819,13 @@
},
"grok-4": {
"promptPrice": 0.002,
- "completionPrice": 0.008
+ "completionPrice": 0.008,
+ "cacheReadPrice": 0.0003
},
"grok-4-fast": {
"promptPrice": 0.0005,
- "completionPrice": 0.0015
+ "completionPrice": 0.0015,
+ "cacheReadPrice": 7.5e-05
},
"grok-vision-beta": {
"promptPrice": 0.0005,
@@ -777,61 +843,109 @@
"promptPrice": 0.00088,
"completionPrice": 0.00088
},
- "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo-vLLM-json":{
+ "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo-vLLM-json": {
"promptPrice": 0.00088,
"completionPrice": 0.00088
},
- "meta-llama/Meta-Llama-3.1-405B-Instruct-Lite-Pro-lora":{
+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Lite-Pro-lora": {
"promptPrice": 0.00088,
"completionPrice": 0.00088
},
- "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo":{
+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": {
"promptPrice": 0.00088,
"completionPrice": 0.00088
},
- "Qwen/QwQ-32B-Preview":{
+ "Qwen/QwQ-32B-Preview": {
"promptPrice": 0.0012,
"completionPrice": 0.0012
},
- "codellama/CodeLlama-34b-Instruct-hf":{
+ "codellama/CodeLlama-34b-Instruct-hf": {
"promptPrice": 0.0008,
- "completionPrice":0.0008
+ "completionPrice": 0.0008
},
- "databricks/dbrx-instruct":{
+ "databricks/dbrx-instruct": {
"promptPrice": 0.0012,
"completionPrice": 0.0012
},
- "deepseek-ai/deepseek-llm-67b-chat":{
+ "deepseek-ai/deepseek-llm-67b-chat": {
"promptPrice": 0.0009,
"completionPrice": 0.0009
},
- "google/gemma-2b-it":{
+ "google/gemma-2b-it": {
"promptPrice": 0.0001,
"completionPrice": 0.0001
},
- "google/gemma-2-27b-it":{
+ "google/gemma-2-27b-it": {
"promptPrice": 0.0008,
"completionPrice": 0.0008
},
- "google/gemma-2-9b-it":{
+ "google/gemma-2-9b-it": {
"promptPrice": 0.0003,
"completionPrice": 0.0003
},
- "Gryphe/MythoMax-L2-13b-Lite":{
+ "Gryphe/MythoMax-L2-13b-Lite": {
"promptPrice": 0.0003,
"completionPrice": 0.0003
},
- "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF":{
+ "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": {
"promptPrice": 0.0088,
"completionPrice": 0.0088
},
- "deepseek-chat":{
+ "deepseek-chat": {
"promptPrice": 0.00014,
"completionPrice": 0.00028
},
- "deepseek-reasoner":{
+ "deepseek-reasoner": {
"promptPrice": 0.00055,
"completionPrice": 0.00219
+ },
+ "gpt-5.6-sol": {
+ "promptPrice": 0.005,
+ "completionPrice": 0.03,
+ "cacheReadPrice": 0.0005,
+ "cacheCreationPrice": 0.00625
+ },
+ "gpt-5.6-terra": {
+ "promptPrice": 0.0025,
+ "completionPrice": 0.015,
+ "cacheReadPrice": 0.00025,
+ "cacheCreationPrice": 0.003125
+ },
+ "gpt-5.6-luna": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.006,
+ "cacheReadPrice": 0.0001,
+ "cacheCreationPrice": 0.00125
+ },
+ "moonshotai/kimi-k2-instruct": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.003,
+ "cacheReadPrice": 0.0005
+ },
+ "moonshotai/kimi-k2-instruct-0905": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.003,
+ "cacheReadPrice": 0.0005
+ },
+ "openai/gpt-oss-20b": {
+ "promptPrice": 7.5e-05,
+ "completionPrice": 0.0003,
+ "cacheReadPrice": 3.75e-05
+ },
+ "openai/gpt-oss-120b": {
+ "promptPrice": 0.00015,
+ "completionPrice": 0.0006,
+ "cacheReadPrice": 7.5e-05
+ },
+ "grok-4.5": {
+ "promptPrice": 0.002,
+ "completionPrice": 0.006,
+ "cacheReadPrice": 0.0003
+ },
+ "grok-4.3": {
+ "promptPrice": 0.00125,
+ "completionPrice": 0.0025,
+ "cacheReadPrice": 0.0002
}
}
-}
\ No newline at end of file
+}
diff --git a/sdk/python/src/openlit/instrumentation/ai21/utils.py b/sdk/python/src/openlit/instrumentation/ai21/utils.py
index d0b8a8d4..d3934163 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/utils.py
@@ -345,7 +345,15 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
@@ -675,7 +683,13 @@ def common_chat_rag_logic(
request_model = scope._kwargs.get("model", "jamba-1.5-mini")
cost = get_chat_model_cost(
- request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ request_model,
+ pricing_info,
+ scope._input_tokens,
+ scope._output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
)
# Common Span Attributes
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
index 91275c4b..34449491 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
@@ -469,7 +469,16 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ # Azure AI Inference prompt_tokens are inclusive of cached tokens.
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/utils.py b/sdk/python/src/openlit/instrumentation/bedrock/utils.py
index 065d27ae..26642143 100644
--- a/sdk/python/src/openlit/instrumentation/bedrock/utils.py
+++ b/sdk/python/src/openlit/instrumentation/bedrock/utils.py
@@ -410,7 +410,16 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ # Bedrock reports inputTokens exclusive of cache read/write tokens.
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=False,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
index 5a4ff9ca..1383c4d6 100644
--- a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
+++ b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
@@ -332,8 +332,8 @@ def set_chat_span_attributes(
pricing_info,
input_tokens,
output_tokens,
- cache_read_tokens,
- cache_creation_tokens,
+ cache_read_tokens=cache_read_tokens,
+ cache_creation_tokens=cache_creation_tokens,
)
span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
@@ -454,8 +454,7 @@ def _calculate_cost(
(uncached input + cache-read + cache-creation), so the cache token counts
are forwarded with ``prompt_tokens_include_cache=True``. This re-prices the
cached tokens at their dedicated cache rates instead of the full prompt
- rate; without it, cache-read tokens (billed at 1/10th of prompt price for
- Claude models) are charged at full prompt price.
+ rate.
"""
if not pricing_info or not model_str:
return 0
@@ -464,8 +463,8 @@ def _calculate_cost(
pricing_info,
input_tokens,
output_tokens,
- cache_read_tokens=cache_read_tokens,
- cache_creation_tokens=cache_creation_tokens,
+ cache_read_tokens=cache_read_tokens or 0,
+ cache_creation_tokens=cache_creation_tokens or 0,
prompt_tokens_include_cache=True,
)
diff --git a/sdk/python/src/openlit/instrumentation/cohere/utils.py b/sdk/python/src/openlit/instrumentation/cohere/utils.py
index 43329054..26c7bc9c 100644
--- a/sdk/python/src/openlit/instrumentation/cohere/utils.py
+++ b/sdk/python/src/openlit/instrumentation/cohere/utils.py
@@ -311,10 +311,12 @@ def process_chunk(scope, chunk):
# Handle token usage including reasoning tokens and cached tokens
scope._input_tokens = billed.get("input_tokens", 0)
scope._output_tokens = billed.get("output_tokens", 0)
- scope._cache_read_input_tokens = getattr(scope, "_cache_read_input_tokens", 0)
- scope._cache_creation_input_tokens = getattr(
- scope, "_cache_creation_input_tokens", 0
- )
+ scope._cache_read_input_tokens = (
+ usage.get("cached_tokens")
+ or (delta.get("meta") or {}).get("cached_tokens")
+ or 0
+ ) or 0
+ scope._cache_creation_input_tokens = 0
scope._end_time = time.time()
@@ -340,8 +342,15 @@ def common_chat_logic(
request_model = scope._kwargs.get("model", "command-r-plus-08-2024")
+ # Cohere billed input is inclusive of cached_tokens when present.
cost = get_chat_model_cost(
- request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ request_model,
+ pricing_info,
+ scope._input_tokens,
+ scope._output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
)
# Common Span Attributes
@@ -649,7 +658,11 @@ def process_chat_response(
# Handle token usage including reasoning tokens and cached tokens
scope._input_tokens = billed.get("input_tokens", 0)
scope._output_tokens = billed.get("output_tokens", 0)
- scope._cache_read_input_tokens = 0 # Cohere does not expose cached tokens in API
+ scope._cache_read_input_tokens = (
+ usage.get("cached_tokens")
+ or (response_dict.get("meta") or {}).get("cached_tokens")
+ or 0
+ ) or 0
scope._cache_creation_input_tokens = 0
scope._timestamps = []
scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0
diff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/utils.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/utils.py
index 5de255b5..4e2f4596 100644
--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/utils.py
+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/utils.py
@@ -519,7 +519,16 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ # Gemini usage_metadata.prompt_token_count includes cached_content_token_count.
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/groq/utils.py b/sdk/python/src/openlit/instrumentation/groq/utils.py
index e8047500..cc55ecc9 100644
--- a/sdk/python/src/openlit/instrumentation/groq/utils.py
+++ b/sdk/python/src/openlit/instrumentation/groq/utils.py
@@ -341,7 +341,16 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ # Groq prompt_tokens are inclusive of cached_tokens.
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.py
index ffc08bad..54d0bad0 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py
@@ -28,6 +28,43 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
+
+def _extract_litellm_cache_tokens(usage):
+ """
+ Extract cache read / creation token counts from LiteLLM usage.
+
+ LiteLLM normalizes OpenAI- and Anthropic-shaped usage. Cache creation must
+ never be taken from ``completion_tokens_details.cached_tokens`` (that field
+ is unrelated). Prefer, in order: top-level ``cache_creation_input_tokens``,
+ ``prompt_tokens_details.cache_creation_tokens`` / ``cache_write_tokens``,
+ then Anthropic ``cache_write_input_tokens``.
+ """
+ usage = usage or {}
+ prompt_details = (
+ usage.get("prompt_tokens_details")
+ or usage.get("input_tokens_details")
+ or {}
+ )
+ if not isinstance(prompt_details, dict):
+ prompt_details = {}
+
+ cache_read = (
+ prompt_details.get("cached_tokens")
+ or usage.get("cache_read_input_tokens")
+ or 0
+ ) or 0
+
+ cache_creation = (
+ usage.get("cache_creation_input_tokens")
+ or prompt_details.get("cache_creation_tokens")
+ or prompt_details.get("cache_write_tokens")
+ or usage.get("cache_write_input_tokens")
+ or 0
+ ) or 0
+
+ return cache_read, cache_creation
+
+
LITELLM_PROVIDER_HOSTS = {
"openai": "api.openai.com",
"azure": "azure.openai.com",
@@ -328,23 +365,6 @@ def emit_inference_event(
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
-def _extract_cache_tokens(usage):
- """Return ``(cache_read, cache_creation)`` from a litellm usage payload.
-
- litellm normalizes both cache counts into ``prompt_tokens_details``: the
- OpenAI-style ``cached_tokens`` is the cache-read count and, for Anthropic,
- ``cache_creation_tokens`` is the cache-write count. The cache-creation count
- was previously read from ``completion_tokens_details.cached_tokens``, a
- field litellm never populates, so it was always 0.
- """
- prompt_tokens_details = (
- usage.get("prompt_tokens_details", usage.get("input_tokens_details", {})) or {}
- )
- cache_read = prompt_tokens_details.get("cached_tokens", 0) or 0
- cache_creation = prompt_tokens_details.get("cache_creation_tokens", 0) or 0
- return cache_read, cache_creation
-
-
def process_chunk(scope, chunk):
"""
Process a chunk of response data and update state.
@@ -420,7 +440,7 @@ def process_chunk(scope, chunk):
(
scope._cache_read_input_tokens,
scope._cache_creation_input_tokens,
- ) = _extract_cache_tokens(usage)
+ ) = _extract_litellm_cache_tokens(usage)
scope._end_time = time.time()
@@ -446,17 +466,15 @@ def common_chat_logic(
request_model = scope._kwargs.get("model", "openai/gpt-4o")
- # litellm normalizes ``prompt_tokens`` to include cache-read and
- # cache-creation tokens (OpenAI convention), so forward the cache counts
- # with ``prompt_tokens_include_cache=True`` to re-price them at their cache
- # rates instead of the full prompt rate.
+ # LiteLLM prompt_tokens are inclusive of cache tokens (OpenAI-style /
+ # normalized Anthropic totals), so subtract when re-pricing cache.
cost = get_chat_model_cost(
request_model,
pricing_info,
scope._input_tokens,
scope._output_tokens,
- cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0),
- cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0),
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
prompt_tokens_include_cache=True,
)
@@ -787,7 +805,7 @@ def process_chat_response(
(
scope._cache_read_input_tokens,
scope._cache_creation_input_tokens,
- ) = _extract_cache_tokens(usage)
+ ) = _extract_litellm_cache_tokens(usage)
scope._response_id = response_dict.get("id") or ""
scope._response_model = response_dict.get("model") or ""
choices = response_dict.get("choices", [])
diff --git a/sdk/python/src/openlit/instrumentation/mistral/utils.py b/sdk/python/src/openlit/instrumentation/mistral/utils.py
index 0787b225..ffde6a8a 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/utils.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/utils.py
@@ -377,9 +377,15 @@ def common_chat_logic(
request_model = scope._kwargs.get("model", "mistral-small-latest")
- # Compute cost
+ # Compute cost (Mistral prompt_tokens include cached_tokens when present).
cost = get_chat_model_cost(
- request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ request_model,
+ pricing_info,
+ scope._input_tokens,
+ scope._output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
)
# Common Span Attributes
diff --git a/sdk/python/src/openlit/instrumentation/openai/utils.py b/sdk/python/src/openlit/instrumentation/openai/utils.py
index 0d93ad24..911bcf8f 100644
--- a/sdk/python/src/openlit/instrumentation/openai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/openai/utils.py
@@ -741,6 +741,15 @@ def process_chat_chunk(scope, chunk):
if usage:
scope._input_tokens = usage.get("prompt_tokens", 0)
scope._output_tokens = usage.get("completion_tokens", 0)
+ prompt_tokens_details = usage.get("prompt_tokens_details") or {}
+ if not isinstance(prompt_tokens_details, dict):
+ prompt_tokens_details = {}
+ scope._cache_read_input_tokens = (
+ prompt_tokens_details.get("cached_tokens", 0) or 0
+ )
+ scope._cache_creation_input_tokens = (
+ prompt_tokens_details.get("cache_write_tokens", 0) or 0
+ )
scope._system_fingerprint = (
chunked.get("system_fingerprint") or scope._system_fingerprint
@@ -850,8 +859,16 @@ def process_response_chunk(scope, chunk):
scope._reasoning_tokens = output_tokens_details.get("reasoning_tokens", 0)
# Cached tokens (OTel: gen_ai.usage.cache_read.input_tokens)
- input_tokens_details = usage.get("input_tokens_details", {})
- scope._cache_read_input_tokens = input_tokens_details.get("cached_tokens", 0)
+ # Cache writes (OTel: gen_ai.usage.cache_creation.input_tokens)
+ input_tokens_details = usage.get("input_tokens_details", {}) or {}
+ if not isinstance(input_tokens_details, dict):
+ input_tokens_details = {}
+ scope._cache_read_input_tokens = (
+ input_tokens_details.get("cached_tokens", 0) or 0
+ )
+ scope._cache_creation_input_tokens = (
+ input_tokens_details.get("cache_write_tokens", 0) or 0
+ )
def common_response_logic(
@@ -887,17 +904,15 @@ def common_response_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- # OpenAI reports prompt_tokens inclusive of cached (cache read) tokens, so
- # flag the prompt tokens as cache-inclusive to avoid billing cached tokens
- # twice once a model defines cacheReadPrice. OpenAI has no cache-creation
- # charge, so that count stays 0 unless a provider sets it.
+ # OpenAI reports prompt_tokens inclusive of cached (cache read) tokens and
+ # cache_write_tokens, so flag the prompt tokens as cache-inclusive.
cost = get_chat_model_cost(
request_model,
pricing_info,
input_tokens,
output_tokens,
- cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0),
- cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0),
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
prompt_tokens_include_cache=True,
)
@@ -1322,8 +1337,13 @@ def process_response_response(
output_tokens_details = usage.get("output_tokens_details", {})
scope._reasoning_tokens = output_tokens_details.get("reasoning_tokens", 0)
- input_tokens_details = usage.get("input_tokens_details", {})
- scope._cache_read_input_tokens = input_tokens_details.get("cached_tokens", 0)
+ input_tokens_details = usage.get("input_tokens_details", {}) or {}
+ if not isinstance(input_tokens_details, dict):
+ input_tokens_details = {}
+ scope._cache_read_input_tokens = input_tokens_details.get("cached_tokens", 0) or 0
+ scope._cache_creation_input_tokens = (
+ input_tokens_details.get("cache_write_tokens", 0) or 0
+ )
scope._timestamps = []
scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0
@@ -1392,17 +1412,15 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- # OpenAI reports prompt_tokens inclusive of cached (cache read) tokens, so
- # flag the prompt tokens as cache-inclusive to avoid billing cached tokens
- # twice once a model defines cacheReadPrice. OpenAI has no cache-creation
- # charge, so that count stays 0 unless a provider sets it.
+ # OpenAI reports prompt_tokens inclusive of cached (cache read) tokens and
+ # cache_write_tokens, so flag the prompt tokens as cache-inclusive.
cost = get_chat_model_cost(
request_model,
pricing_info,
input_tokens,
output_tokens,
- cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0),
- cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0),
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
prompt_tokens_include_cache=True,
)
@@ -1766,10 +1784,15 @@ def process_chat_response(
scope._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)
# Extract cache tokens (OpenAI prompt caching)
- prompt_tokens_details = response_dict.get("usage", {}).get(
- "prompt_tokens_details", {}
+ prompt_tokens_details = (
+ response_dict.get("usage", {}).get("prompt_tokens_details", {}) or {}
+ )
+ if not isinstance(prompt_tokens_details, dict):
+ prompt_tokens_details = {}
+ scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) or 0
+ scope._cache_creation_input_tokens = (
+ prompt_tokens_details.get("cache_write_tokens", 0) or 0
)
- scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
scope._timestamps = []
scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0
diff --git a/sdk/python/src/openlit/instrumentation/premai/utils.py b/sdk/python/src/openlit/instrumentation/premai/utils.py
index 83f7dd87..9214d633 100644
--- a/sdk/python/src/openlit/instrumentation/premai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/premai/utils.py
@@ -346,7 +346,15 @@ def common_chat_logic(
input_tokens = scope._input_tokens
output_tokens = scope._output_tokens
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/pydo/utils.py b/sdk/python/src/openlit/instrumentation/pydo/utils.py
index ab2dda67..66b24262 100644
--- a/sdk/python/src/openlit/instrumentation/pydo/utils.py
+++ b/sdk/python/src/openlit/instrumentation/pydo/utils.py
@@ -594,8 +594,15 @@ def common_chat_logic(
if not scope._output_tokens:
scope._output_tokens = general_tokens(scope._llmresponse or "")
+ # pydo sums Anthropic cache tokens into input_tokens (inclusive total).
cost = get_chat_model_cost(
- request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ request_model,
+ pricing_info,
+ scope._input_tokens,
+ scope._output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
)
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/reka/utils.py b/sdk/python/src/openlit/instrumentation/reka/utils.py
index 9e3af386..0d25db71 100644
--- a/sdk/python/src/openlit/instrumentation/reka/utils.py
+++ b/sdk/python/src/openlit/instrumentation/reka/utils.py
@@ -298,7 +298,15 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/sarvam/utils.py b/sdk/python/src/openlit/instrumentation/sarvam/utils.py
index 789c11bb..c5531f70 100644
--- a/sdk/python/src/openlit/instrumentation/sarvam/utils.py
+++ b/sdk/python/src/openlit/instrumentation/sarvam/utils.py
@@ -282,8 +282,16 @@ def common_chat_logic(
input_tokens = getattr(scope, "_input_tokens", 0)
output_tokens = getattr(scope, "_output_tokens", 0)
- # Compute cost
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ # Compute cost (Sarvam chat prompt_tokens include cached_tokens when present).
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
+ )
# Common Span Attributes
common_span_attributes(
diff --git a/sdk/python/src/openlit/instrumentation/strands/processor.py b/sdk/python/src/openlit/instrumentation/strands/processor.py
index 53df7007..5bf952b5 100644
--- a/sdk/python/src/openlit/instrumentation/strands/processor.py
+++ b/sdk/python/src/openlit/instrumentation/strands/processor.py
@@ -414,8 +414,44 @@ class StrandsSpanProcessor(SpanProcessor):
input_tokens
) + int(output_tokens)
if self._pricing_info and model_name:
+ cache_read = (
+ attrs.get(SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)
+ or attrs.get("gen_ai.usage.cache_read_input_tokens")
+ or 0
+ )
+ cache_creation = (
+ attrs.get(SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS)
+ or attrs.get("gen_ai.usage.cache_write_input_tokens")
+ or 0
+ )
+ try:
+ cache_read = int(cache_read or 0)
+ except (TypeError, ValueError):
+ cache_read = 0
+ try:
+ cache_creation = int(cache_creation or 0)
+ except (TypeError, ValueError):
+ cache_creation = 0
+ try:
+ input_tokens_int = int(input_tokens or 0)
+ except (TypeError, ValueError):
+ input_tokens_int = 0
+ try:
+ output_tokens_int = int(output_tokens or 0)
+ except (TypeError, ValueError):
+ output_tokens_int = 0
+ # Strands may expose Anthropic-exclusive or OpenAI-inclusive totals.
+ prompt_tokens_include_cache = input_tokens_int >= (
+ cache_read + cache_creation
+ )
cost = get_chat_model_cost(
- model_name, self._pricing_info, input_tokens, output_tokens
+ model_name,
+ self._pricing_info,
+ input_tokens_int,
+ output_tokens_int,
+ cache_read_tokens=cache_read,
+ cache_creation_tokens=cache_creation,
+ prompt_tokens_include_cache=prompt_tokens_include_cache,
)
enrichments[SemanticConvention.GEN_AI_USAGE_COST] = cost
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/utils.py b/sdk/python/src/openlit/instrumentation/vertexai/utils.py
index 50c0181e..1e6b0cec 100644
--- a/sdk/python/src/openlit/instrumentation/vertexai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/vertexai/utils.py
@@ -327,8 +327,15 @@ def common_chat_logic(
input_tokens = general_tokens(prompt)
output_tokens = general_tokens(scope._llmresponse)
+ # Vertex AI prompt_token_count includes cached_content_token_count.
cost = get_chat_model_cost(
- scope._request_model, pricing_info, input_tokens, output_tokens
+ scope._request_model,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0) or 0,
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0) or 0,
+ prompt_tokens_include_cache=True,
)
# Common Span Attributes
diff --git a/sdk/python/tests/test_cache_pricing.py b/sdk/python/tests/test_cache_pricing.py
index f4fad162..6804a4f9 100644
--- a/sdk/python/tests/test_cache_pricing.py
+++ b/sdk/python/tests/test_cache_pricing.py
@@ -5,8 +5,8 @@ Unit tests for cache-aware cost calculation in ``get_chat_model_cost``.
These cover the two token-accounting conventions used by the instrumentations
that forward cache token counts:
-* Anthropic native API -> ``prompt_tokens`` is *exclusive* of cache tokens.
-* LangChain ``usage_metadata`` -> ``prompt_tokens`` is *inclusive* of cache
+* Anthropic native API / Bedrock -> ``prompt_tokens`` is *exclusive* of cache tokens.
+* LangChain / LiteLLM / OpenAI-style -> ``prompt_tokens`` is *inclusive* of cache
tokens (sum of all input token types).
The same cached tokens must never be billed twice, and models without cache
@@ -16,6 +16,7 @@ pricing must produce exactly the legacy cost regardless of convention.
import pytest
from openlit.__helpers import get_chat_model_cost
+from openlit.instrumentation.litellm.utils import _extract_litellm_cache_tokens
PRICING_WITH_CACHE = {
@@ -84,6 +85,26 @@ class TestAnthropicConvention:
)
assert cost == _approx((800 / 1000) * 0.003 + (500 / 1000) * 0.015)
+ def test_bedrock_style_exclusive_cache_fields(self):
+ # Bedrock Converse: inputTokens exclusive; cacheReadInputTokens /
+ # cacheWriteInputTokens passed as read / creation with include=False.
+ cost = get_chat_model_cost(
+ MODEL,
+ PRICING_WITH_CACHE,
+ prompt_tokens=800,
+ completion_tokens=200,
+ cache_read_tokens=150,
+ cache_creation_tokens=50,
+ prompt_tokens_include_cache=False,
+ )
+ expected = (
+ (800 / 1000) * 0.003
+ + (200 / 1000) * 0.015
+ + (150 / 1000) * 0.0003
+ + (50 / 1000) * 0.00375
+ )
+ assert cost == _approx(expected)
+
class TestLangChainConvention:
"""prompt_tokens already includes cache tokens (sum of all input types)."""
@@ -167,6 +188,27 @@ class TestLangChainConvention:
expected = (600 / 1000) * 0.0025 + (200 / 1000) * 0.01 + (400 / 1000) * 0.00125
assert cost == _approx(expected)
+ def test_litellm_style_inclusive_with_creation(self):
+ # LiteLLM-normalized Anthropic/OpenAI usage: prompt_tokens already
+ # include cache buckets; creation comes from cache_creation_input_tokens
+ # / prompt_tokens_details.cache_write_tokens (never completion details).
+ cost = get_chat_model_cost(
+ MODEL,
+ PRICING_WITH_CACHE,
+ prompt_tokens=1200,
+ completion_tokens=100,
+ cache_read_tokens=200,
+ cache_creation_tokens=200,
+ prompt_tokens_include_cache=True,
+ )
+ expected = (
+ (800 / 1000) * 0.003
+ + (100 / 1000) * 0.015
+ + (200 / 1000) * 0.0003
+ + (200 / 1000) * 0.00375
+ )
+ assert cost == _approx(expected)
+
def test_cache_exceeding_prompt_does_not_go_negative(self):
# Defensive: malformed data where cache tokens exceed reported input.
cost = get_chat_model_cost(
@@ -183,6 +225,52 @@ class TestLangChainConvention:
assert cost == _approx(expected)
+class TestLiteLLMCacheExtraction:
+ def test_prefers_cache_creation_input_tokens(self):
+ read, creation = _extract_litellm_cache_tokens(
+ {
+ "prompt_tokens_details": {
+ "cached_tokens": 10,
+ "cache_write_tokens": 99,
+ },
+ "cache_creation_input_tokens": 42,
+ "completion_tokens_details": {"cached_tokens": 999},
+ }
+ )
+ assert read == 10
+ assert creation == 42
+
+ def test_falls_back_to_prompt_details_write_tokens(self):
+ read, creation = _extract_litellm_cache_tokens(
+ {
+ "prompt_tokens_details": {
+ "cached_tokens": 5,
+ "cache_write_tokens": 7,
+ },
+ "completion_tokens_details": {"cached_tokens": 999},
+ }
+ )
+ assert read == 5
+ assert creation == 7
+
+ def test_anthropic_top_level_fields(self):
+ read, creation = _extract_litellm_cache_tokens(
+ {
+ "cache_read_input_tokens": 11,
+ "cache_creation_input_tokens": 13,
+ }
+ )
+ assert read == 11
+ assert creation == 13
+
+ def test_ignores_completion_tokens_details_cached_tokens(self):
+ read, creation = _extract_litellm_cache_tokens(
+ {"completion_tokens_details": {"cached_tokens": 999}}
+ )
+ assert read == 0
+ assert creation == 0
+
+
class TestBundledPricingFile:
"""Sanity-check the cache fields actually shipped in assets/pricing.json."""
@@ -298,9 +386,7 @@ class TestLiteLLMAdapter:
}
def test_extract_cache_tokens_reads_prompt_tokens_details(self):
- from openlit.instrumentation.litellm.utils import _extract_cache_tokens
-
- cache_read, cache_creation = _extract_cache_tokens(self.USAGE)
+ cache_read, cache_creation = _extract_litellm_cache_tokens(self.USAGE)
assert cache_read == _CACHE_READ
# Was always 0 when read from completion_tokens_details.cached_tokens.
assert cache_creation == _CACHE_CREATION
diff --git a/sdk/typescript/src/instrumentation/__tests__/bedrock-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/bedrock-trace-comparison.test.ts
index bfcc0ecf..ac009337 100644
--- a/sdk/typescript/src/instrumentation/__tests__/bedrock-trace-comparison.test.ts
+++ b/sdk/typescript/src/instrumentation/__tests__/bedrock-trace-comparison.test.ts
@@ -479,7 +479,7 @@ describe('BedrockWrapper', () => {
isStream: false,
});
- expect(costSpy).toHaveBeenCalledWith('test-model', mockPricingInfo, 100, 50);
+ expect(costSpy).toHaveBeenCalledWith('test-model', mockPricingInfo, 100, 50, 0, 0, false);
});
it('should set TTFT and TBT when provided for streaming', () => {
diff --git a/sdk/typescript/src/instrumentation/__tests__/cache-pricing.test.ts b/sdk/typescript/src/instrumentation/__tests__/cache-pricing.test.ts
index 2e431f6c..ecd31144 100644
--- a/sdk/typescript/src/instrumentation/__tests__/cache-pricing.test.ts
+++ b/sdk/typescript/src/instrumentation/__tests__/cache-pricing.test.ts
@@ -8,6 +8,9 @@
*
* The same cached tokens must never be billed twice, and models without cache
* pricing must produce exactly the legacy cost regardless of convention.
+ *
+ * Also covers instrumentor call-site patterns (claude-agent-sdk inclusive sum,
+ * bedrock exclusive, openai cache_write_tokens, strands/cursor heuristic).
*/
import OpenLitHelper from '../../helpers';
@@ -24,11 +27,33 @@ const PRICING = {
promptPrice: 0.003,
completionPrice: 0.015,
},
+ 'gpt-cache-write': {
+ promptPrice: 0.0025,
+ completionPrice: 0.01,
+ cacheReadPrice: 0.00125,
+ cacheCreationPrice: 0.003125,
+ },
},
};
const MODEL = 'claude-3-5-sonnet-20241022';
+// 200 uncached + 5000 cache-read + 1000 cache-creation + 300 output
+const RAW_INPUT = 200;
+const CACHE_READ = 5000;
+const CACHE_CREATION = 1000;
+const OUTPUT = 300;
+const INCLUSIVE_INPUT = RAW_INPUT + CACHE_READ + CACHE_CREATION;
+
+function groundTruth(): number {
+ return (
+ (RAW_INPUT / 1000) * 0.003 +
+ (OUTPUT / 1000) * 0.015 +
+ (CACHE_READ / 1000) * 0.0003 +
+ (CACHE_CREATION / 1000) * 0.00375
+ );
+}
+
describe('OpenLitHelper.getChatModelCost — cache pricing', () => {
describe('no cache tokens', () => {
it('matches the legacy formula', () => {
@@ -116,4 +141,98 @@ describe('OpenLitHelper.getChatModelCost — cache pricing', () => {
expect(cost).toBeCloseTo(expected, 10);
});
});
+
+ describe('instrumentor call-site patterns', () => {
+ it('claude-agent-sdk: inclusive sum + cache args re-prices correctly', () => {
+ // extractUsage builds input = raw + cache_read + cache_creation
+ const cost = OpenLitHelper.getChatModelCost(
+ MODEL,
+ PRICING,
+ INCLUSIVE_INPUT,
+ OUTPUT,
+ CACHE_READ,
+ CACHE_CREATION,
+ true
+ );
+ expect(cost).toBeCloseTo(groundTruth(), 10);
+ });
+
+ it('bedrock: exclusive cacheRead/cacheWrite does not undercharge', () => {
+ const cost = OpenLitHelper.getChatModelCost(
+ MODEL,
+ PRICING,
+ RAW_INPUT,
+ OUTPUT,
+ CACHE_READ,
+ CACHE_CREATION,
+ false
+ );
+ expect(cost).toBeCloseTo(groundTruth(), 10);
+ });
+
+ it('openai: maps cache_write_tokens into creation (inclusive)', () => {
+ // prompt_tokens inclusive; cache_write_tokens billed at cacheCreationPrice
+ const cost = OpenLitHelper.getChatModelCost(
+ 'gpt-cache-write',
+ PRICING,
+ 1000,
+ 200,
+ 400,
+ 100,
+ true
+ );
+ const expected =
+ (500 / 1000) * 0.0025 +
+ (200 / 1000) * 0.01 +
+ (400 / 1000) * 0.00125 +
+ (100 / 1000) * 0.003125;
+ expect(cost).toBeCloseTo(expected, 10);
+ });
+
+ it('strands/cursor heuristic: input >= cache sum → inclusive', () => {
+ const cacheTotal = CACHE_READ + CACHE_CREATION;
+ const inclusive = INCLUSIVE_INPUT >= cacheTotal;
+ expect(inclusive).toBe(true);
+ const cost = OpenLitHelper.getChatModelCost(
+ MODEL,
+ PRICING,
+ INCLUSIVE_INPUT,
+ OUTPUT,
+ CACHE_READ,
+ CACHE_CREATION,
+ inclusive
+ );
+ expect(cost).toBeCloseTo(groundTruth(), 10);
+ });
+
+ it('strands/cursor heuristic: input < cache sum → exclusive', () => {
+ const cacheTotal = CACHE_READ + CACHE_CREATION;
+ const inclusive = RAW_INPUT >= cacheTotal;
+ expect(inclusive).toBe(false);
+ const cost = OpenLitHelper.getChatModelCost(
+ MODEL,
+ PRICING,
+ RAW_INPUT,
+ OUTPUT,
+ CACHE_READ,
+ CACHE_CREATION,
+ inclusive
+ );
+ expect(cost).toBeCloseTo(groundTruth(), 10);
+ });
+
+ it('billing inclusive total at prompt price overcharges (documents pre-fix)', () => {
+ const overcharged = OpenLitHelper.getChatModelCost(
+ MODEL,
+ PRICING,
+ INCLUSIVE_INPUT,
+ OUTPUT
+ );
+ expect(overcharged).toBeCloseTo(
+ (INCLUSIVE_INPUT / 1000) * 0.003 + (OUTPUT / 1000) * 0.015,
+ 10
+ );
+ expect(overcharged).toBeGreaterThan(groundTruth());
+ });
+ });
});
diff --git a/sdk/typescript/src/instrumentation/__tests__/groq-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/groq-trace-comparison.test.ts
index c4255e6b..a45ac4c2 100644
--- a/sdk/typescript/src/instrumentation/__tests__/groq-trace-comparison.test.ts
+++ b/sdk/typescript/src/instrumentation/__tests__/groq-trace-comparison.test.ts
@@ -526,7 +526,10 @@ describe('Groq Cross-Language Trace Comparison', () => {
'llama-3.1-8b-instant',
expect.any(Object),
100,
- 50
+ 50,
+ 0,
+ 0,
+ true
);
});
});
diff --git a/sdk/typescript/src/instrumentation/azure-ai-inference/wrapper.ts b/sdk/typescript/src/instrumentation/azure-ai-inference/wrapper.ts
index c3775d74..2442687c 100644
--- a/sdk/typescript/src/instrumentation/azure-ai-inference/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/azure-ai-inference/wrapper.ts
@@ -459,11 +459,22 @@ class AzureAIInferenceWrapper extends BaseWrapper {
const pricingInfo = OpenlitConfig.pricingInfo || {};
const inputTokens = result.usage?.prompt_tokens || 0;
const outputTokens = result.usage?.completion_tokens || 0;
+ // Azure AI Inference follows OpenAI-style inclusive prompt_tokens.
+ const cacheReadTokens =
+ Number(result.usage?.prompt_tokens_details?.cached_tokens) || 0;
+ const cacheCreationTokens =
+ Number(result.usage?.input_tokens_details?.cache_creation_tokens) ||
+ Number(result.usage?.prompt_tokens_details?.cache_creation_tokens) ||
+ Number(result.usage?.prompt_tokens_details?.cache_write_tokens) ||
+ 0;
const cost = OpenLitHelper.getChatModelCost(
requestModel,
pricingInfo,
inputTokens,
- outputTokens
+ outputTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ true
);
AzureAIInferenceWrapper.setBaseSpanAttributes(span, {
@@ -480,16 +491,16 @@ class AzureAIInferenceWrapper extends BaseWrapper {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
- if (result.usage?.prompt_tokens_details?.cached_tokens) {
+ if (cacheReadTokens) {
span.setAttribute(
SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
- result.usage.prompt_tokens_details.cached_tokens
+ cacheReadTokens
);
}
- if (result.usage?.input_tokens_details?.cache_creation_tokens) {
+ if (cacheCreationTokens) {
span.setAttribute(
SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
- result.usage.input_tokens_details.cache_creation_tokens
+ cacheCreationTokens
);
}
diff --git a/sdk/typescript/src/instrumentation/bedrock/wrapper.ts b/sdk/typescript/src/instrumentation/bedrock/wrapper.ts
index c1b10604..32f23e48 100644
--- a/sdk/typescript/src/instrumentation/bedrock/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/bedrock/wrapper.ts
@@ -381,8 +381,17 @@ class BedrockWrapper extends BaseWrapper {
const responseModel = modelId;
const finishReason = mapFinishReason(result.stopReason || 'stop');
+ // Bedrock Converse reports inputTokens exclusive of cache read/write.
const pricingInfo = OpenlitConfig.pricingInfo || {};
- const cost = OpenLitHelper.getChatModelCost(modelId, pricingInfo, inputTokens, outputTokens);
+ const cost = OpenLitHelper.getChatModelCost(
+ modelId,
+ pricingInfo,
+ inputTokens,
+ outputTokens,
+ cacheReadTokens,
+ cacheWriteTokens,
+ false
+ );
BedrockWrapper.setBaseSpanAttributes(span, {
genAIEndpoint,
diff --git a/sdk/typescript/src/instrumentation/claude-agent-sdk/wrapper.ts b/sdk/typescript/src/instrumentation/claude-agent-sdk/wrapper.ts
index 0861cb93..9d2b8be5 100644
--- a/sdk/typescript/src/instrumentation/claude-agent-sdk/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/claude-agent-sdk/wrapper.ts
@@ -602,8 +602,24 @@ function setChatSpanAttributes(
const inputTokens = usageAttrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] ?? 0;
const outputTokens = usageAttrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] ?? 0;
+ // extractUsage builds an inclusive input total (uncached + cache-read +
+ // cache-creation), so re-price cache tokens with promptTokensIncludeCache.
+ const cacheReadTokens =
+ usageAttrs[SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] ?? 0;
+ const cacheCreationTokens =
+ usageAttrs[SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] ?? 0;
const pricingInfo = OpenlitConfig.pricingInfo || {};
- const cost = model ? OpenLitHelper.getChatModelCost(model, pricingInfo, inputTokens, outputTokens) : 0;
+ const cost = model
+ ? OpenLitHelper.getChatModelCost(
+ model,
+ pricingInfo,
+ inputTokens,
+ outputTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ true
+ )
+ : 0;
span.setAttribute(SemanticConvention.GEN_AI_USAGE_COST, cost);
let outputMessages: any[] | null = null;
diff --git a/sdk/typescript/src/instrumentation/cursor-sdk/wrapper.ts b/sdk/typescript/src/instrumentation/cursor-sdk/wrapper.ts
index 6240749b..23605dbf 100644
--- a/sdk/typescript/src/instrumentation/cursor-sdk/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/cursor-sdk/wrapper.ts
@@ -711,8 +711,21 @@ function createRunProxy(
const pricingInfo = OpenlitConfig.pricingInfo || {};
const effectiveModel = responseModel || requestModel;
+ // Cursor reports cache write separately; treat it as cache-creation for pricing.
+ // Prefer inclusive accounting when input already covers the cache sum.
+ const cacheTotal = (usage.cacheReadTokens || 0) + (usage.cacheWriteTokens || 0);
+ const promptTokensIncludeCache =
+ cacheTotal > 0 && usage.inputTokens >= cacheTotal;
const cost = effectiveModel
- ? OpenLitHelper.getChatModelCost(effectiveModel, pricingInfo, usage.inputTokens, usage.outputTokens)
+ ? OpenLitHelper.getChatModelCost(
+ effectiveModel,
+ pricingInfo,
+ usage.inputTokens,
+ usage.outputTokens,
+ usage.cacheReadTokens,
+ usage.cacheWriteTokens,
+ promptTokensIncludeCache
+ )
: 0;
if (cost) span.setAttribute(SemanticConvention.GEN_AI_USAGE_COST, cost);
diff --git a/sdk/typescript/src/instrumentation/google-ai/wrapper.ts b/sdk/typescript/src/instrumentation/google-ai/wrapper.ts
index 8a557937..b331bb23 100644
--- a/sdk/typescript/src/instrumentation/google-ai/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/google-ai/wrapper.ts
@@ -337,12 +337,18 @@ class GoogleAIWrapper extends BaseWrapper {
const usageMetadata = responseData.usageMetadata;
const inputTokens = usageMetadata?.promptTokenCount || 0;
const outputTokens = usageMetadata?.candidatesTokenCount || 0;
+ // Gemini promptTokenCount includes cached content tokens.
+ const cacheReadTokens = usageMetadata?.cachedContentTokenCount || 0;
+ const cacheCreationTokens = usageMetadata?.cacheCreationInputTokens || 0;
const cost = OpenLitHelper.getChatModelCost(
requestModel,
pricingInfo,
inputTokens,
- outputTokens
+ outputTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ true
);
GoogleAIWrapper.setBaseSpanAttributes(span, {
@@ -408,11 +414,9 @@ class GoogleAIWrapper extends BaseWrapper {
}
// Cache tokens (matches Python: cached_content_token_count, cache_creation_input_tokens)
- const cacheReadTokens = usageMetadata?.cachedContentTokenCount || 0;
if (cacheReadTokens) {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cacheReadTokens);
}
- const cacheCreationTokens = usageMetadata?.cacheCreationInputTokens || 0;
if (cacheCreationTokens) {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, cacheCreationTokens);
}
diff --git a/sdk/typescript/src/instrumentation/groq/wrapper.ts b/sdk/typescript/src/instrumentation/groq/wrapper.ts
index 0e30db05..39a78170 100644
--- a/sdk/typescript/src/instrumentation/groq/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/groq/wrapper.ts
@@ -142,6 +142,12 @@ class GroqWrapper extends BaseWrapper {
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
+ prompt_tokens_details: {
+ cached_tokens: 0,
+ },
+ input_tokens_details: {
+ cache_creation_tokens: 0,
+ },
},
};
@@ -204,6 +210,18 @@ class GroqWrapper extends BaseWrapper {
result.usage.prompt_tokens = usage.prompt_tokens || 0;
result.usage.completion_tokens = usage.completion_tokens || 0;
result.usage.total_tokens = usage.total_tokens || 0;
+ if (usage.prompt_tokens_details) {
+ result.usage.prompt_tokens_details = {
+ ...(result.usage.prompt_tokens_details || {}),
+ ...usage.prompt_tokens_details,
+ };
+ }
+ if (usage.input_tokens_details) {
+ result.usage.input_tokens_details = {
+ ...(result.usage.input_tokens_details || {}),
+ ...usage.input_tokens_details,
+ };
+ }
}
yield chunk;
@@ -228,6 +246,7 @@ class GroqWrapper extends BaseWrapper {
);
if (completionTokens) {
result.usage = {
+ ...result.usage,
prompt_tokens: promptTokens,
completion_tokens: completionTokens,
total_tokens: promptTokens + completionTokens,
@@ -329,11 +348,25 @@ class GroqWrapper extends BaseWrapper {
const pricingInfo = OpenlitConfig.pricingInfo || {};
+ // Groq reports prompt_tokens inclusive of cached tokens (OpenAI-compat).
+ const cacheReadTokens =
+ Number(result.usage?.prompt_tokens_details?.cached_tokens) ||
+ Number(result.usage?.cached_tokens) ||
+ 0;
+ const cacheCreationTokens =
+ Number(result.usage?.input_tokens_details?.cache_creation_tokens) ||
+ Number(result.usage?.prompt_tokens_details?.cache_creation_tokens) ||
+ Number(result.usage?.prompt_tokens_details?.cache_write_tokens) ||
+ 0;
+
const cost = OpenLitHelper.getChatModelCost(
requestModel,
pricingInfo,
result.usage.prompt_tokens,
- result.usage.completion_tokens
+ result.usage.completion_tokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ true
);
GroqWrapper.setBaseSpanAttributes(span, {
@@ -357,6 +390,16 @@ class GroqWrapper extends BaseWrapper {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+ if (cacheReadTokens) {
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cacheReadTokens);
+ }
+ if (cacheCreationTokens) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
+ cacheCreationTokens
+ );
+ }
+
if (ttft > 0) {
span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);
}
diff --git a/sdk/typescript/src/instrumentation/openai/wrapper.ts b/sdk/typescript/src/instrumentation/openai/wrapper.ts
index 429bd9cc..95f935cd 100644
--- a/sdk/typescript/src/instrumentation/openai/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/openai/wrapper.ts
@@ -196,6 +196,8 @@ class OpenAIWrapper extends BaseWrapper {
},
prompt_tokens_details: {
cached_tokens: 0,
+ cache_creation_tokens: 0,
+ cache_write_tokens: 0,
audio_tokens: 0,
},
},
@@ -258,6 +260,21 @@ class OpenAIWrapper extends BaseWrapper {
tools = true;
}
+ if (chunk.usage) {
+ result.usage = {
+ ...result.usage,
+ ...chunk.usage,
+ prompt_tokens_details: {
+ ...result.usage.prompt_tokens_details,
+ ...(chunk.usage.prompt_tokens_details || {}),
+ },
+ completion_tokens_details: {
+ ...result.usage.completion_tokens_details,
+ ...(chunk.usage.completion_tokens_details || {}),
+ },
+ };
+ }
+
yield chunk;
}
@@ -268,23 +285,25 @@ class OpenAIWrapper extends BaseWrapper {
} as any;
}
- let promptTokens = 0;
- for (const message of messages || []) {
- promptTokens += OpenLitHelper.openaiTokens(message.content as string, result.model) ?? 0;
- }
+ if (!result.usage.prompt_tokens && !result.usage.completion_tokens) {
+ let promptTokens = 0;
+ for (const message of messages || []) {
+ promptTokens += OpenLitHelper.openaiTokens(message.content as string, result.model) ?? 0;
+ }
- const completionTokens = OpenLitHelper.openaiTokens(
- result.choices[0].message.content ?? '',
- result.model
- );
- if (completionTokens) {
- result.usage = {
- prompt_tokens: promptTokens,
- completion_tokens: completionTokens,
- total_tokens: promptTokens + completionTokens,
- completion_tokens_details: result.usage.completion_tokens_details,
- prompt_tokens_details: result.usage.prompt_tokens_details,
- };
+ const completionTokens = OpenLitHelper.openaiTokens(
+ result.choices[0].message.content ?? '',
+ result.model
+ );
+ if (completionTokens) {
+ result.usage = {
+ prompt_tokens: promptTokens,
+ completion_tokens: completionTokens,
+ total_tokens: promptTokens + completionTokens,
+ completion_tokens_details: result.usage.completion_tokens_details,
+ prompt_tokens_details: result.usage.prompt_tokens_details,
+ };
+ }
}
args[0].tools = tools;
@@ -387,14 +406,21 @@ class OpenAIWrapper extends BaseWrapper {
// OpenAI reports prompt_tokens inclusive of cached (cache read) tokens, so
// flag the prompt tokens as cache-inclusive to avoid billing cached tokens
- // twice once a model defines cacheReadPrice.
+ // twice once a model defines cacheReadPrice. GPT-5.6+ also reports
+ // cache_write_tokens (creation) under prompt_tokens_details.
+ const promptDetails = result.usage.prompt_tokens_details || {};
+ const cacheReadTokens = Number(promptDetails.cached_tokens) || 0;
+ const cacheCreationTokens =
+ Number(promptDetails.cache_creation_tokens) ||
+ Number(promptDetails.cache_write_tokens) ||
+ 0;
const cost = OpenLitHelper.getChatModelCost(
requestModel,
pricingInfo,
result.usage.prompt_tokens,
result.usage.completion_tokens,
- Number(result.usage.prompt_tokens_details?.cached_tokens) || 0,
- Number(result.usage.prompt_tokens_details?.cache_creation_tokens) || 0,
+ cacheReadTokens,
+ cacheCreationTokens,
true
);
@@ -423,16 +449,16 @@ class OpenAIWrapper extends BaseWrapper {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
- if (result.usage.prompt_tokens_details?.cached_tokens) {
+ if (cacheReadTokens) {
span.setAttribute(
SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
- result.usage.prompt_tokens_details.cached_tokens
+ cacheReadTokens
);
}
- if (result.usage.prompt_tokens_details?.cache_creation_tokens) {
+ if (cacheCreationTokens) {
span.setAttribute(
SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
- result.usage.prompt_tokens_details.cache_creation_tokens
+ cacheCreationTokens
);
}
@@ -1046,6 +1072,11 @@ class OpenAIWrapper extends BaseWrapper {
usage: {
input_tokens: 0,
output_tokens: 0,
+ input_tokens_details: {
+ cached_tokens: 0,
+ cache_creation_tokens: 0,
+ cache_write_tokens: 0,
+ },
output_tokens_details: {
reasoning_tokens: 0,
},
@@ -1088,6 +1119,10 @@ class OpenAIWrapper extends BaseWrapper {
const usage = responseData.usage || {};
result.usage.input_tokens = usage.input_tokens || 0;
result.usage.output_tokens = usage.output_tokens || 0;
+ result.usage.input_tokens_details = {
+ ...(result.usage.input_tokens_details || {}),
+ ...(usage.input_tokens_details || {}),
+ };
result.usage.output_tokens_details.reasoning_tokens =
usage.output_tokens_details?.reasoning_tokens || 0;
}
@@ -1187,14 +1222,21 @@ class OpenAIWrapper extends BaseWrapper {
const inputTokens = result.usage?.input_tokens || 0;
const outputTokens = result.usage?.output_tokens || 0;
- // Responses API reports input_tokens inclusive of cached tokens.
+ // Responses API reports input_tokens inclusive of cached tokens. Map
+ // cache_write_tokens / cache_creation_tokens into creation for pricing.
+ const inputDetails = result.usage?.input_tokens_details || {};
+ const cacheReadTokens = Number(inputDetails.cached_tokens) || 0;
+ const cacheCreationTokens =
+ Number(inputDetails.cache_creation_tokens) ||
+ Number(inputDetails.cache_write_tokens) ||
+ 0;
const cost = OpenLitHelper.getChatModelCost(
requestModel,
pricingInfo,
inputTokens,
outputTokens,
- Number(result.usage?.input_tokens_details?.cached_tokens) || 0,
- 0,
+ cacheReadTokens,
+ cacheCreationTokens,
true
);
@@ -1220,6 +1262,19 @@ class OpenAIWrapper extends BaseWrapper {
span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+
+ if (cacheReadTokens) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
+ cacheReadTokens
+ );
+ }
+ if (cacheCreationTokens) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
+ cacheCreationTokens
+ );
+ }
if (result.usage?.output_tokens_details?.reasoning_tokens) {
span.setAttribute(
diff --git a/sdk/typescript/src/instrumentation/strands/processor.ts b/sdk/typescript/src/instrumentation/strands/processor.ts
index 309b659c..3815bca4 100644
--- a/sdk/typescript/src/instrumentation/strands/processor.ts
+++ b/sdk/typescript/src/instrumentation/strands/processor.ts
@@ -400,7 +400,29 @@ export class StrandsSpanProcessor implements SpanProcessor {
enrichments[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] = inputTokens + outputTokens;
}
if (OpenlitConfig.pricingInfo && modelName) {
- const cost = OpenLitHelper.getChatModelCost(modelName, OpenlitConfig.pricingInfo, inputTokens, outputTokens);
+ const cacheReadTokens = Number(
+ attrs[SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] ||
+ attrs['gen_ai.usage.cache_read_input_tokens'] ||
+ 0
+ );
+ const cacheCreationTokens = Number(
+ attrs[SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] ||
+ attrs['gen_ai.usage.cache_write_input_tokens'] ||
+ 0
+ );
+ const cacheTotal = cacheReadTokens + cacheCreationTokens;
+ // Underlying providers differ; treat input as inclusive when it already
+ // covers the cache sum, otherwise exclusive (Anthropic/Bedrock-style).
+ const promptTokensIncludeCache = cacheTotal > 0 && inputTokens >= cacheTotal;
+ const cost = OpenLitHelper.getChatModelCost(
+ modelName,
+ OpenlitConfig.pricingInfo,
+ inputTokens,
+ outputTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ promptTokensIncludeCache
+ );
enrichments[SemanticConvention.GEN_AI_USAGE_COST] = cost;
}
diff --git a/sdk/typescript/src/instrumentation/vertexai/wrapper.ts b/sdk/typescript/src/instrumentation/vertexai/wrapper.ts
index 97ef51a6..41b45503 100644
--- a/sdk/typescript/src/instrumentation/vertexai/wrapper.ts
+++ b/sdk/typescript/src/instrumentation/vertexai/wrapper.ts
@@ -450,8 +450,17 @@ class VertexAIWrapper extends BaseWrapper {
const cacheReadTokens = usageMetadata?.cachedContentTokenCount || 0;
const cacheCreationTokens = usageMetadata?.cacheCreationInputTokens || 0;
+ // Vertex/Gemini promptTokenCount includes cached content tokens.
const pricingInfo = OpenlitConfig.pricingInfo || {};
- const cost = OpenLitHelper.getChatModelCost(requestModel, pricingInfo, inputTokens, outputTokens);
+ const cost = OpenLitHelper.getChatModelCost(
+ requestModel,
+ pricingInfo,
+ inputTokens,
+ outputTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ true
+ );
VertexAIWrapper.setBaseSpanAttributes(span, {
genAIEndpoint,
diff --git a/src/client/src/__tests__/lib/platform/pricing/chat-cost.test.ts b/src/client/src/__tests__/lib/platform/pricing/chat-cost.test.ts
new file mode 100644
index 00000000..fb0f40a5
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/pricing/chat-cost.test.ts
@@ -0,0 +1,73 @@
+import {
+ getChatModelCostPerM,
+ promptTokensIncludeCache,
+} from "@/lib/platform/pricing/chat-cost";
+
+describe("getChatModelCostPerM", () => {
+ const claudePrices = {
+ inputPricePerMToken: 3,
+ outputPricePerMToken: 15,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
+ };
+
+ it("matches legacy input+output when cache prices are zero", () => {
+ const cost = getChatModelCostPerM(
+ {
+ inputPricePerMToken: 2.5,
+ outputPricePerMToken: 10,
+ },
+ { promptTokens: 1_000_000, completionTokens: 1_000_000 }
+ );
+ expect(cost).toBeCloseTo(12.5, 8);
+ });
+
+ it("prices exclusive Anthropic-style cache tokens on top of prompt", () => {
+ // 200 uncached + 5000 cache read + 1000 cache create + 300 out
+ const cost = getChatModelCostPerM(
+ claudePrices,
+ {
+ promptTokens: 200,
+ completionTokens: 300,
+ cacheReadTokens: 5000,
+ cacheCreationTokens: 1000,
+ },
+ { promptTokensIncludeCache: false }
+ );
+ // 200/1e6*3 + 300/1e6*15 + 5000/1e6*0.3 + 1000/1e6*3.75
+ expect(cost).toBeCloseTo(0.01035, 8);
+ });
+
+ it("subtracts cache from inclusive prompt totals", () => {
+ const cost = getChatModelCostPerM(
+ claudePrices,
+ {
+ promptTokens: 6200,
+ completionTokens: 300,
+ cacheReadTokens: 5000,
+ cacheCreationTokens: 1000,
+ },
+ { promptTokensIncludeCache: true }
+ );
+ expect(cost).toBeCloseTo(0.01035, 8);
+ });
+});
+
+describe("promptTokensIncludeCache", () => {
+ it("treats anthropic and bedrock as exclusive", () => {
+ expect(promptTokensIncludeCache("anthropic", 6200, 5000, 1000)).toBe(
+ false
+ );
+ expect(promptTokensIncludeCache("bedrock", 6200, 5000, 1000)).toBe(false);
+ });
+
+ it("treats openai and litellm as inclusive", () => {
+ expect(promptTokensIncludeCache("openai", 6200, 5000, 1000)).toBe(true);
+ expect(promptTokensIncludeCache("litellm", 6200, 5000, 1000)).toBe(true);
+ });
+
+ it("falls back to heuristic when provider unknown", () => {
+ expect(promptTokensIncludeCache("unknown", 6200, 5000, 1000)).toBe(true);
+ expect(promptTokensIncludeCache("unknown", 200, 5000, 1000)).toBe(false);
+ });
+});
diff --git a/src/client/src/app/api/openground/models/export/route.ts b/src/client/src/app/api/openground/models/export/route.ts
index 8e6d4531..d8339bee 100644
--- a/src/client/src/app/api/openground/models/export/route.ts
+++ b/src/client/src/app/api/openground/models/export/route.ts
@@ -39,7 +39,9 @@ export async function GET(request: NextRequest) {
model_id,
model_type,
input_price_per_m_token as inputPrice,
- output_price_per_m_token as outputPrice
+ output_price_per_m_token as outputPrice,
+ cache_read_price_per_m_token as cacheReadPrice,
+ cache_creation_price_per_m_token as cacheCreationPrice
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
ORDER BY model_type, model_id
`;
@@ -66,10 +68,17 @@ export async function GET(request: NextRequest) {
}
if (type === "chat") {
- pricing[type][model.model_id] = {
+ const entry: Record<string, number> = {
promptPrice: model.inputPrice / 1000, // per-M to per-K
completionPrice: model.outputPrice / 1000,
};
+ if (model.cacheReadPrice > 0) {
+ entry.cacheReadPrice = model.cacheReadPrice / 1000;
+ }
+ if (model.cacheCreationPrice > 0) {
+ entry.cacheCreationPrice = model.cacheCreationPrice / 1000;
+ }
+ pricing[type][model.model_id] = entry;
} else if (type === "embeddings") {
pricing[type][model.model_id] = model.inputPrice / 1000;
} else if (type === "audio") {
diff --git a/src/client/src/app/api/openground/models/import/route.ts b/src/client/src/app/api/openground/models/import/route.ts
index a5035315..81994e2d 100644
--- a/src/client/src/app/api/openground/models/import/route.ts
+++ b/src/client/src/app/api/openground/models/import/route.ts
@@ -72,6 +72,8 @@ async function POSTHandler(request: NextRequest) {
context_window: number;
input_price_per_m_token: number;
output_price_per_m_token: number;
+ cache_read_price_per_m_token: number;
+ cache_creation_price_per_m_token: number;
capabilities: string[];
is_default: boolean;
created_by_user_id: string;
@@ -87,6 +89,12 @@ async function POSTHandler(request: NextRequest) {
context_window: m.contextWindow || m.context_window || 4096,
input_price_per_m_token: m.inputPricePerMToken || m.input_price_per_m_token || 0,
output_price_per_m_token: m.outputPricePerMToken || m.output_price_per_m_token || 0,
+ cache_read_price_per_m_token:
+ m.cacheReadPricePerMToken || m.cache_read_price_per_m_token || 0,
+ cache_creation_price_per_m_token:
+ m.cacheCreationPricePerMToken ||
+ m.cache_creation_price_per_m_token ||
+ 0,
capabilities: m.capabilities || [],
is_default: false,
created_by_user_id: session.user!.id!,
@@ -100,6 +108,8 @@ async function POSTHandler(request: NextRequest) {
for (const [modelId, pricing] of Object.entries(models as Record<string, any>)) {
let inputPrice = 0;
let outputPrice = 0;
+ let cacheReadPrice = 0;
+ let cacheCreationPrice = 0;
if (typeof pricing === "number") {
// embeddings / audio format
@@ -108,6 +118,8 @@ async function POSTHandler(request: NextRequest) {
if ("promptPrice" in pricing) {
inputPrice = (pricing.promptPrice || 0) * 1000;
outputPrice = (pricing.completionPrice || 0) * 1000;
+ cacheReadPrice = (pricing.cacheReadPrice || 0) * 1000;
+ cacheCreationPrice = (pricing.cacheCreationPrice || 0) * 1000;
} else if ("standard" in pricing) {
// images format
const firstRes = Object.values(pricing.standard || {})[0];
@@ -123,6 +135,8 @@ async function POSTHandler(request: NextRequest) {
context_window: 4096,
input_price_per_m_token: inputPrice,
output_price_per_m_token: outputPrice,
+ cache_read_price_per_m_token: cacheReadPrice,
+ cache_creation_price_per_m_token: cacheCreationPrice,
capabilities: [],
is_default: false,
created_by_user_id: session.user!.id!,
diff --git a/src/client/src/app/api/openground/models/route.ts b/src/client/src/app/api/openground/models/route.ts
index 28eee521..195dce31 100644
--- a/src/client/src/app/api/openground/models/route.ts
+++ b/src/client/src/app/api/openground/models/route.ts
@@ -57,6 +57,8 @@ async function GETHandler(request: NextRequest) {
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities,
is_default as isDefault
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
@@ -100,6 +102,8 @@ async function GETHandler(request: NextRequest) {
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities,
is_default as isDefault
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
@@ -170,6 +174,10 @@ async function POSTHandler(request: NextRequest) {
context_window = ${model.contextWindow || 4096},
input_price_per_m_token = ${model.inputPricePerMToken || 0},
output_price_per_m_token = ${model.outputPricePerMToken || 0},
+ cache_read_price_per_m_token = ${model.cacheReadPricePerMToken || 0},
+ cache_creation_price_per_m_token = ${
+ model.cacheCreationPricePerMToken || 0
+ },
capabilities = [${capabilitiesArray}],
updated_at = now()
WHERE model_id = '${Sanitizer.sanitizeValue(modelId)}'
@@ -213,6 +221,9 @@ async function POSTHandler(request: NextRequest) {
context_window: model.contextWindow || 4096,
input_price_per_m_token: model.inputPricePerMToken || 0,
output_price_per_m_token: model.outputPricePerMToken || 0,
+ cache_read_price_per_m_token: model.cacheReadPricePerMToken || 0,
+ cache_creation_price_per_m_token:
+ model.cacheCreationPricePerMToken || 0,
capabilities: model.capabilities || [],
is_default: false,
created_by_user_id: session.user.id,
@@ -245,6 +256,8 @@ async function POSTHandler(request: NextRequest) {
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities,
is_default as isDefault
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
diff --git a/src/client/src/app/api/pricing/export/[dbConfigId]/route.ts b/src/client/src/app/api/pricing/export/[dbConfigId]/route.ts
index da9bb60b..f91556f8 100644
--- a/src/client/src/app/api/pricing/export/[dbConfigId]/route.ts
+++ b/src/client/src/app/api/pricing/export/[dbConfigId]/route.ts
@@ -41,7 +41,9 @@ export async function GET(
model_id,
model_type,
input_price_per_m_token as inputPrice,
- output_price_per_m_token as outputPrice
+ output_price_per_m_token as outputPrice,
+ cache_read_price_per_m_token as cacheReadPrice,
+ cache_creation_price_per_m_token as cacheCreationPrice
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
ORDER BY model_type, model_id
`;
@@ -71,10 +73,17 @@ export async function GET(
}
if (type === "chat") {
- pricing[type][model.model_id] = {
+ const entry: Record<string, number> = {
promptPrice: model.inputPrice / 1000,
completionPrice: model.outputPrice / 1000,
};
+ if (model.cacheReadPrice > 0) {
+ entry.cacheReadPrice = model.cacheReadPrice / 1000;
+ }
+ if (model.cacheCreationPrice > 0) {
+ entry.cacheCreationPrice = model.cacheCreationPrice / 1000;
+ }
+ pricing[type][model.model_id] = entry;
} else if (type === "embeddings") {
pricing[type][model.model_id] = model.inputPrice / 1000;
} else if (type === "audio") {
diff --git a/src/client/src/clickhouse/migrations/add-provider-models-cache-prices-migration.ts b/src/client/src/clickhouse/migrations/add-provider-models-cache-prices-migration.ts
new file mode 100644
index 00000000..961081a7
--- /dev/null
+++ b/src/client/src/clickhouse/migrations/add-provider-models-cache-prices-migration.ts
@@ -0,0 +1,26 @@
+import migrationHelper from "./migration-helper";
+import { OPENLIT_PROVIDER_MODELS_TABLE_NAME } from "@/lib/platform/providers/table-details";
+
+const MIGRATION_ID = "add-provider-models-cache-prices";
+
+/**
+ * Adds optional cache-read / cache-creation price columns to
+ * openlit_provider_models so Manage Models and auto-pricing can bill
+ * prompt-cache tokens at dedicated rates.
+ */
+export default async function AddProviderModelsCachePricesMigration(
+ databaseConfigId?: string
+) {
+ const queries = [
+ `ALTER TABLE ${OPENLIT_PROVIDER_MODELS_TABLE_NAME} ADD COLUMN IF NOT EXISTS cache_read_price_per_m_token Float64 DEFAULT 0 AFTER output_price_per_m_token`,
+ `ALTER TABLE ${OPENLIT_PROVIDER_MODELS_TABLE_NAME} ADD COLUMN IF NOT EXISTS cache_creation_price_per_m_token Float64 DEFAULT 0 AFTER cache_read_price_per_m_token`,
+ ];
+
+ const { migrationExist, queriesRun } = await migrationHelper({
+ clickhouseMigrationId: MIGRATION_ID,
+ databaseConfigId,
+ queries,
+ });
+
+ return { migrationExist, queriesRun };
+}
diff --git a/src/client/src/clickhouse/migrations/create-providers-migration.ts b/src/client/src/clickhouse/migrations/create-providers-migration.ts
index 25006c56..381ef9c8 100644
--- a/src/client/src/clickhouse/migrations/create-providers-migration.ts
+++ b/src/client/src/clickhouse/migrations/create-providers-migration.ts
@@ -83,6 +83,8 @@ export default async function CreateProvidersMigration(databaseConfigId?: string
context_window UInt32 DEFAULT 4096,
input_price_per_m_token Float64 DEFAULT 0,
output_price_per_m_token Float64 DEFAULT 0,
+ cache_read_price_per_m_token Float64 DEFAULT 0,
+ cache_creation_price_per_m_token Float64 DEFAULT 0,
capabilities Array(String) DEFAULT [],
is_default Boolean DEFAULT false,
@@ -152,6 +154,9 @@ export default async function CreateProvidersMigration(databaseConfigId?: string
context_window: model.contextWindow,
input_price_per_m_token: model.inputPricePerMToken,
output_price_per_m_token: model.outputPricePerMToken,
+ cache_read_price_per_m_token: model.cacheReadPricePerMToken || 0,
+ cache_creation_price_per_m_token:
+ model.cacheCreationPricePerMToken || 0,
capabilities: model.capabilities || [],
is_default: true,
created_by_user_id: "",
diff --git a/src/client/src/clickhouse/migrations/index.ts b/src/client/src/clickhouse/migrations/index.ts
index c1ec5df4..3dc1e38f 100644
--- a/src/client/src/clickhouse/migrations/index.ts
+++ b/src/client/src/clickhouse/migrations/index.ts
@@ -33,6 +33,7 @@ import AddCodingAgentSummaryFieldsMigration from "./add-coding-agent-summary-fie
import AddCodingAgentLOCSummaryFieldsMigration from "./add-coding-agent-loc-summary-fields-migration";
import CreateCodingAgentsAuditMigration from "./create-coding-agents-audit-migration";
import DropVcsMigration from "./drop-vcs-migration";
+import AddProviderModelsCachePricesMigration from "./add-provider-models-cache-prices-migration";
export default async function migrations(databaseConfigId?: string) {
// Group 1: Independent table creations (safe to parallel)
@@ -65,6 +66,7 @@ export default async function migrations(databaseConfigId?: string) {
// Group 3: Provider migrations (sequential -- metadata depends on providers)
await CreateProvidersMigration(databaseConfigId);
+ await AddProviderModelsCachePricesMigration(databaseConfigId);
await Promise.all([
CreateProviderMetadataMigration(databaseConfigId),
DropLegacyOpengroundTablesMigration(databaseConfigId),
diff --git a/src/client/src/components/(playground)/openground/model-editor-panel.tsx b/src/client/src/components/(playground)/openground/model-editor-panel.tsx
index 54b77a1a..f2271ae3 100644
--- a/src/client/src/components/(playground)/openground/model-editor-panel.tsx
+++ b/src/client/src/components/(playground)/openground/model-editor-panel.tsx
@@ -58,6 +58,8 @@ export default function ModelEditorPanel({
contextWindow: 4096,
inputPricePerMToken: 0,
outputPricePerMToken: 0,
+ cacheReadPricePerMToken: 0,
+ cacheCreationPricePerMToken: 0,
capabilities: [],
});
@@ -76,6 +78,8 @@ export default function ModelEditorPanel({
contextWindow: model.contextWindow,
inputPricePerMToken: model.inputPricePerMToken,
outputPricePerMToken: model.outputPricePerMToken,
+ cacheReadPricePerMToken: model.cacheReadPricePerMToken || 0,
+ cacheCreationPricePerMToken: model.cacheCreationPricePerMToken || 0,
capabilities: model.capabilities || [],
});
} else if (isAddingNew) {
@@ -87,6 +91,8 @@ export default function ModelEditorPanel({
contextWindow: 4096,
inputPricePerMToken: 0,
outputPricePerMToken: 0,
+ cacheReadPricePerMToken: 0,
+ cacheCreationPricePerMToken: 0,
capabilities: [],
});
}
@@ -109,6 +115,8 @@ export default function ModelEditorPanel({
contextWindow: formData.contextWindow || 4096,
inputPricePerMToken: formData.inputPricePerMToken || 0,
outputPricePerMToken: formData.outputPricePerMToken || 0,
+ cacheReadPricePerMToken: formData.cacheReadPricePerMToken || 0,
+ cacheCreationPricePerMToken: formData.cacheCreationPricePerMToken || 0,
capabilities: formData.capabilities || [],
},
customId: formData.id || undefined,
@@ -257,6 +265,42 @@ export default function ModelEditorPanel({
onChange={(e) => setFormData({ ...formData, outputPricePerMToken: parseFloat(e.target.value) || 0 })}
/>
</div>
+ <div className="space-y-2">
+ <Label htmlFor="cache-read-price">
+ {getMessage().OPENGROUND_CACHE_READ_PRICE_PER_M_TOKENS}
+ </Label>
+ <Input
+ id="cache-read-price"
+ type="number"
+ step="0.001"
+ placeholder="0"
+ value={formData.cacheReadPricePerMToken}
+ onChange={(e) =>
+ setFormData({
+ ...formData,
+ cacheReadPricePerMToken: parseFloat(e.target.value) || 0,
+ })
+ }
+ />
+ </div>
+ <div className="space-y-2">
+ <Label htmlFor="cache-creation-price">
+ {getMessage().OPENGROUND_CACHE_CREATION_PRICE_PER_M_TOKENS}
+ </Label>
+ <Input
+ id="cache-creation-price"
+ type="number"
+ step="0.001"
+ placeholder="0"
+ value={formData.cacheCreationPricePerMToken}
+ onChange={(e) =>
+ setFormData({
+ ...formData,
+ cacheCreationPricePerMToken: parseFloat(e.target.value) || 0,
+ })
+ }
+ />
+ </div>
</div>
{/* Capabilities */}
diff --git a/src/client/src/components/(playground)/openground/model-management-dialog.tsx b/src/client/src/components/(playground)/openground/model-management-dialog.tsx
index 954e1a98..a0de48c2 100644
--- a/src/client/src/components/(playground)/openground/model-management-dialog.tsx
+++ b/src/client/src/components/(playground)/openground/model-management-dialog.tsx
@@ -57,6 +57,8 @@ export default function ModelManagementDialog({
contextWindow: 4096,
inputPricePerMToken: 0,
outputPricePerMToken: 0,
+ cacheReadPricePerMToken: 0,
+ cacheCreationPricePerMToken: 0,
capabilities: [],
});
@@ -90,6 +92,8 @@ export default function ModelManagementDialog({
contextWindow: formData.contextWindow || 4096,
inputPricePerMToken: formData.inputPricePerMToken || 0,
outputPricePerMToken: formData.outputPricePerMToken || 0,
+ cacheReadPricePerMToken: formData.cacheReadPricePerMToken || 0,
+ cacheCreationPricePerMToken: formData.cacheCreationPricePerMToken || 0,
capabilities: formData.capabilities || [],
},
customId: editingModel?.customId,
@@ -144,6 +148,8 @@ export default function ModelManagementDialog({
contextWindow: model.contextWindow,
inputPricePerMToken: model.inputPricePerMToken,
outputPricePerMToken: model.outputPricePerMToken,
+ cacheReadPricePerMToken: model.cacheReadPricePerMToken || 0,
+ cacheCreationPricePerMToken: model.cacheCreationPricePerMToken || 0,
capabilities: model.capabilities || [],
});
setIsEditing(true);
@@ -156,6 +162,8 @@ export default function ModelManagementDialog({
contextWindow: 4096,
inputPricePerMToken: 0,
outputPricePerMToken: 0,
+ cacheReadPricePerMToken: 0,
+ cacheCreationPricePerMToken: 0,
capabilities: [],
});
setIsEditing(false);
@@ -199,17 +207,18 @@ export default function ModelManagementDialog({
</div>
</div>
- <div className="grid grid-cols-3 gap-4">
- <div className="space-y-2">
- <Label htmlFor="context-window">{getMessage().OPENGROUND_CONTEXT_WINDOW}</Label>
- <Input
- id="context-window"
- type="number"
- placeholder="4096"
- value={formData.contextWindow}
- onChange={(e) => setFormData({ ...formData, contextWindow: parseInt(e.target.value) || 0 })}
- />
- </div>
+ <div className="space-y-2">
+ <Label htmlFor="context-window">{getMessage().OPENGROUND_CONTEXT_WINDOW}</Label>
+ <Input
+ id="context-window"
+ type="number"
+ placeholder="4096"
+ value={formData.contextWindow}
+ onChange={(e) => setFormData({ ...formData, contextWindow: parseInt(e.target.value) || 0 })}
+ />
+ </div>
+
+ <div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label htmlFor="input-price">{getMessage().OPENGROUND_INPUT_PRICE_PER_M_TOKENS}</Label>
<Input
@@ -232,6 +241,43 @@ export default function ModelManagementDialog({
onChange={(e) => setFormData({ ...formData, outputPricePerMToken: parseFloat(e.target.value) || 0 })}
/>
</div>
+ <div className="space-y-2">
+ <Label htmlFor="cache-read-price">
+ {getMessage().OPENGROUND_CACHE_READ_PRICE_PER_M_TOKENS}
+ </Label>
+ <Input
+ id="cache-read-price"
+ type="number"
+ step="0.001"
+ placeholder="0"
+ value={formData.cacheReadPricePerMToken}
+ onChange={(e) =>
+ setFormData({
+ ...formData,
+ cacheReadPricePerMToken: parseFloat(e.target.value) || 0,
+ })
+ }
+ />
+ </div>
+ <div className="space-y-2">
+ <Label htmlFor="cache-creation-price">
+ {getMessage().OPENGROUND_CACHE_CREATION_PRICE_PER_M_TOKENS}
+ </Label>
+ <Input
+ id="cache-creation-price"
+ type="number"
+ step="0.001"
+ placeholder="0"
+ value={formData.cacheCreationPricePerMToken}
+ onChange={(e) =>
+ setFormData({
+ ...formData,
+ cacheCreationPricePerMToken:
+ parseFloat(e.target.value) || 0,
+ })
+ }
+ />
+ </div>
</div>
<div className="space-y-2">
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index 5dd2b0bf..4634e89e 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -469,6 +469,10 @@ export const OPENGROUND_MODEL_DISPLAY_NAME = "Display Name";
export const OPENGROUND_CONTEXT_WINDOW = "Context Window";
export const OPENGROUND_INPUT_PRICE_PER_M_TOKENS = "Input Price (per 1M tokens)";
export const OPENGROUND_OUTPUT_PRICE_PER_M_TOKENS = "Output Price (per 1M tokens)";
+export const OPENGROUND_CACHE_READ_PRICE_PER_M_TOKENS =
+ "Cache Read Price (per 1M tokens)";
+export const OPENGROUND_CACHE_CREATION_PRICE_PER_M_TOKENS =
+ "Cache Creation Price (per 1M tokens)";
export const OPENGROUND_MODEL_CAPABILITIES = "Capabilities (comma separated)";
export const OPENGROUND_SAVE_MODEL = "Save Model";
export const OPENGROUND_MODEL_SAVED_SUCCESS = "Model saved successfully!";
@@ -610,7 +614,7 @@ export const PRICING_HOW_AUTO_WORKS_STEP_2 =
export const PRICING_HOW_AUTO_WORKS_STEP_3 =
"Looks up the model: for each span, reads gen_ai.system (provider) + gen_ai.request.model and finds the matching row in openlit_provider_models.";
export const PRICING_HOW_AUTO_WORKS_STEP_4 =
- "Computes cost: (input_tokens / 1M) × input_price_per_m_token + (output_tokens / 1M) × output_price_per_m_token.";
+ "Computes cost from input/output tokens plus optional cache-read and cache-creation tokens at their Manage Models rates (inclusive vs exclusive prompt accounting by provider).";
export const PRICING_HOW_AUTO_WORKS_STEP_5 =
"Writes back: ALTER TABLE otel_traces UPDATE SpanAttributes['gen_ai.usage.cost'] for each processed span. Spans with missing provider/model/tokens or an unknown model are skipped (not errors).";
export const PRICING_HOW_AUTO_WORKS_STEP_6 =
@@ -1782,7 +1786,7 @@ export const CHAT_SETTINGS_CONFIG_OUTPUT = "Output";
export const CHAT_SETTINGS_CONFIG_CONTEXT = "Context";
export const CHAT_SETTINGS_CONFIG_COST_CALCULATION = "Cost calculation";
export const CHAT_SETTINGS_CONFIG_COST_FORMULA =
- "cost = (input_tokens / 1M) x input_price + (output_tokens / 1M) x output_price";
+ "cost = (billable_input / 1M) x input_price + (output / 1M) x output_price + (cache_read / 1M) x cache_read_price + (cache_creation / 1M) x cache_creation_price";
export const CHAT_SETTINGS_OR = "or";
export const CHAT_FAILED_TO_CREATE_CONVERSATION = "Failed to create conversation";
export const CHAT_FAILED_TO_DELETE_CONVERSATION = "Failed to delete conversation";
diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.ts
index 7a2aebfb..17da3fea 100644
--- a/src/client/src/lib/platform/evaluation/index.ts
+++ b/src/client/src/lib/platform/evaluation/index.ts
@@ -46,6 +46,7 @@ import { CronType, CronRunStatus, CronLogData } from "@/types/cron";
import { differenceInSeconds } from "date-fns";
import { getFilterPreviousParams } from "@/helpers/server/platform";
import { ProviderRegistry } from "@/lib/platform/providers/provider-registry";
+import { getChatModelCostPerM } from "@/lib/platform/pricing/chat-cost";
import {
resolveEvaluationType,
type EvaluationTypeRef,
@@ -128,9 +129,15 @@ async function estimateEvaluationCost(
return 0;
}
- return (
- (promptTokens / 1_000_000) * modelMeta.inputPricePerMToken +
- (completionTokens / 1_000_000) * modelMeta.outputPricePerMToken
+ return getChatModelCostPerM(
+ {
+ inputPricePerMToken: modelMeta.inputPricePerMToken,
+ outputPricePerMToken: modelMeta.outputPricePerMToken,
+ cacheReadPricePerMToken: modelMeta.cacheReadPricePerMToken,
+ cacheCreationPricePerMToken: modelMeta.cacheCreationPricePerMToken,
+ },
+ { promptTokens, completionTokens },
+ { promptTokensIncludeCache: true }
);
} catch (e) {
consoleLog("Evaluation cost lookup failed:", e);
diff --git a/src/client/src/lib/platform/openground/evaluate.ts b/src/client/src/lib/platform/openground/evaluate.ts
index 31aa8f67..a0194424 100644
--- a/src/client/src/lib/platform/openground/evaluate.ts
+++ b/src/client/src/lib/platform/openground/evaluate.ts
@@ -1,6 +1,7 @@
import { AISdkAdapter } from "./ai-sdk-adapter";
import { getOpenGroundConfigWithSecret } from "@/lib/platform/providers/config";
import { ProviderRegistry } from "@/lib/platform/providers/provider-registry";
+import { getChatModelCostPerM } from "@/lib/platform/pricing/chat-cost";
import { createOpengroundEvaluation } from "@/lib/platform/openground-clickhouse";
import type { ProviderResult } from "@/lib/platform/openground-clickhouse";
import { getSpecificPrompt } from "@/lib/platform/prompt";
@@ -83,7 +84,9 @@ async function calculateCost(
model: string,
promptTokens: number,
completionTokens: number,
- databaseConfigId: string
+ databaseConfigId: string,
+ cacheReadTokens: number = 0,
+ cacheCreationTokens: number = 0
): Promise<number> {
// Look up the model directly from the openlit_provider_models table
const modelMetadata = await ProviderRegistry.getModel(
@@ -97,11 +100,23 @@ async function calculateCost(
return 0;
}
- const inputCost = (promptTokens / 1_000_000) * modelMetadata.inputPricePerMToken;
- const outputCost =
- (completionTokens / 1_000_000) * modelMetadata.outputPricePerMToken;
-
- return inputCost + outputCost;
+ return getChatModelCostPerM(
+ {
+ inputPricePerMToken: modelMetadata.inputPricePerMToken,
+ outputPricePerMToken: modelMetadata.outputPricePerMToken,
+ cacheReadPricePerMToken: modelMetadata.cacheReadPricePerMToken,
+ cacheCreationPricePerMToken: modelMetadata.cacheCreationPricePerMToken,
+ },
+ {
+ promptTokens,
+ completionTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ },
+ {
+ promptTokensIncludeCache: true,
+ }
+ );
}
/**
diff --git a/src/client/src/lib/platform/pricing/chat-cost.ts b/src/client/src/lib/platform/pricing/chat-cost.ts
new file mode 100644
index 00000000..bd4c5ae1
--- /dev/null
+++ b/src/client/src/lib/platform/pricing/chat-cost.ts
@@ -0,0 +1,123 @@
+/**
+ * Cache-aware chat cost calculation using Manage Models per-million prices.
+ * Mirrors SDK get_chat_model_cost / OpenLitHelper.getChatModelCost semantics.
+ */
+
+export type PerMModelPrices = {
+ inputPricePerMToken: number;
+ outputPricePerMToken: number;
+ cacheReadPricePerMToken?: number;
+ cacheCreationPricePerMToken?: number;
+};
+
+export type ChatCostTokens = {
+ promptTokens: number;
+ completionTokens: number;
+ cacheReadTokens?: number;
+ cacheCreationTokens?: number;
+};
+
+/** Providers that report prompt/input tokens exclusive of cache tokens. */
+const EXCLUSIVE_CACHE_PROVIDERS = new Set([
+ "anthropic",
+ "bedrock",
+ "aws.bedrock",
+ "aws_bedrock",
+ "amazon.bedrock",
+]);
+
+/** Providers that report prompt/input tokens inclusive of cache tokens. */
+const INCLUSIVE_CACHE_PROVIDERS = new Set([
+ "openai",
+ "azure",
+ "azure_ai",
+ "azure.ai",
+ "groq",
+ "google",
+ "google_ai",
+ "google_genai",
+ "vertexai",
+ "vertex_ai",
+ "vertex.ai",
+ "litellm",
+ "langchain",
+ "cohere",
+ "xai",
+ "claude_agent_sdk",
+ "claude-agent-sdk",
+ "pydo",
+]);
+
+/**
+ * Decide whether promptTokens already include cache read/creation counts.
+ */
+export function promptTokensIncludeCache(
+ provider: string,
+ promptTokens: number,
+ cacheReadTokens: number,
+ cacheCreationTokens: number
+): boolean {
+ const normalized = (provider || "").trim().toLowerCase();
+ if (EXCLUSIVE_CACHE_PROVIDERS.has(normalized)) {
+ return false;
+ }
+ if (
+ INCLUSIVE_CACHE_PROVIDERS.has(normalized) ||
+ (normalized.includes("claude") && normalized.includes("agent")) ||
+ normalized.startsWith("google") ||
+ normalized.startsWith("vertex") ||
+ normalized.startsWith("azure")
+ ) {
+ return true;
+ }
+
+ const cacheTotal = (cacheReadTokens || 0) + (cacheCreationTokens || 0);
+ if (cacheTotal > 0 && promptTokens >= cacheTotal) {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Compute USD cost from per-million token prices.
+ * When cache prices are 0 / undefined, behavior matches legacy input+output only
+ * (cache args do not change the result).
+ */
+export function getChatModelCostPerM(
+ prices: PerMModelPrices,
+ tokens: ChatCostTokens,
+ options?: { promptTokensIncludeCache?: boolean }
+): number {
+ const promptTokens = tokens.promptTokens || 0;
+ const completionTokens = tokens.completionTokens || 0;
+ const cacheReadTokens = tokens.cacheReadTokens || 0;
+ const cacheCreationTokens = tokens.cacheCreationTokens || 0;
+
+ const cacheReadPrice = prices.cacheReadPricePerMToken || 0;
+ const cacheCreationPrice = prices.cacheCreationPricePerMToken || 0;
+ const inclusive = options?.promptTokensIncludeCache ?? false;
+
+ let billablePromptTokens = promptTokens;
+ let cacheCost = 0;
+
+ if (cacheReadPrice > 0) {
+ cacheCost += (cacheReadTokens / 1_000_000) * cacheReadPrice;
+ if (inclusive) {
+ billablePromptTokens -= cacheReadTokens;
+ }
+ }
+ if (cacheCreationPrice > 0) {
+ cacheCost += (cacheCreationTokens / 1_000_000) * cacheCreationPrice;
+ if (inclusive) {
+ billablePromptTokens -= cacheCreationTokens;
+ }
+ }
+
+ billablePromptTokens = Math.max(billablePromptTokens, 0);
+
+ return (
+ (billablePromptTokens / 1_000_000) * (prices.inputPricePerMToken || 0) +
+ (completionTokens / 1_000_000) * (prices.outputPricePerMToken || 0) +
+ cacheCost
+ );
+}
diff --git a/src/client/src/lib/platform/pricing/index.ts b/src/client/src/lib/platform/pricing/index.ts
index fb29e7f1..d140ae18 100644
--- a/src/client/src/lib/platform/pricing/index.ts
+++ b/src/client/src/lib/platform/pricing/index.ts
@@ -16,6 +16,10 @@ import { CronRunStatus, CronType } from "@/types/cron";
import { differenceInSeconds } from "date-fns";
import Sanitizer from "@/utils/sanitizer";
import asaw from "@/utils/asaw";
+import {
+ getChatModelCostPerM,
+ promptTokensIncludeCache,
+} from "./chat-cost";
const COST_KEY = getTraceMappingKeyFullPath("cost") as string; // gen_ai.usage.cost
const MODEL_KEY = getTraceMappingKeyFullPath("model") as string; // gen_ai.request.model
@@ -27,6 +31,12 @@ const PROMPT_TOKENS_KEYS = getTraceMappingKeyFullPaths(
const COMPLETION_TOKENS_KEYS = getTraceMappingKeyFullPaths(
"completionTokens"
) as string[];
+const CACHE_READ_TOKENS_KEYS = getTraceMappingKeyFullPaths(
+ "cacheReadTokens"
+) as string[];
+const CACHE_CREATION_TOKENS_KEYS = getTraceMappingKeyFullPaths(
+ "cacheCreationTokens"
+) as string[];
interface TraceRow {
SpanId: string;
@@ -99,11 +109,35 @@ async function computeCostForTrace(
return { cost: null, reason };
}
- const inputCost = (promptTokens / 1_000_000) * modelMeta.inputPricePerMToken;
- const outputCost =
- (completionTokens / 1_000_000) * modelMeta.outputPricePerMToken;
+ const cacheReadTokens = getNumericAttr(trace, CACHE_READ_TOKENS_KEYS);
+ const cacheCreationTokens = getNumericAttr(
+ trace,
+ CACHE_CREATION_TOKENS_KEYS
+ );
+ const inclusive = promptTokensIncludeCache(
+ provider,
+ promptTokens,
+ cacheReadTokens,
+ cacheCreationTokens
+ );
+
+ const cost = getChatModelCostPerM(
+ {
+ inputPricePerMToken: modelMeta.inputPricePerMToken,
+ outputPricePerMToken: modelMeta.outputPricePerMToken,
+ cacheReadPricePerMToken: modelMeta.cacheReadPricePerMToken,
+ cacheCreationPricePerMToken: modelMeta.cacheCreationPricePerMToken,
+ },
+ {
+ promptTokens,
+ completionTokens,
+ cacheReadTokens,
+ cacheCreationTokens,
+ },
+ { promptTokensIncludeCache: inclusive }
+ );
- return { cost: inputCost + outputCost };
+ return { cost };
}
/**
diff --git a/src/client/src/lib/platform/providers/default-models.ts b/src/client/src/lib/platform/providers/default-models.ts
index 5934069e..44430d44 100644
--- a/src/client/src/lib/platform/providers/default-models.ts
+++ b/src/client/src/lib/platform/providers/default-models.ts
@@ -145,6 +145,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 128000,
inputPricePerMToken: 2.5,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -153,6 +154,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 128000,
inputPricePerMToken: 0.15,
outputPricePerMToken: 0.6,
+ cacheReadPricePerMToken: 0.075,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -184,6 +186,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -192,6 +195,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
capabilities: ["function-calling", "streaming"],
},
{
@@ -200,6 +204,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 0.25,
outputPricePerMToken: 2.0,
+ cacheReadPricePerMToken: 0.025,
capabilities: ["function-calling", "streaming"],
},
{
@@ -208,6 +213,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 0.05,
outputPricePerMToken: 0.4,
+ cacheReadPricePerMToken: 0.005,
capabilities: ["function-calling", "streaming"],
},
{
@@ -224,6 +230,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -232,6 +239,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
capabilities: ["function-calling", "streaming"],
},
{
@@ -240,6 +248,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
capabilities: ["function-calling", "streaming"],
},
{
@@ -248,6 +257,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.75,
outputPricePerMToken: 14.0,
+ cacheReadPricePerMToken: 0.175,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -256,6 +266,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.75,
outputPricePerMToken: 14.0,
+ cacheReadPricePerMToken: 0.175,
capabilities: ["function-calling", "streaming"],
},
{
@@ -264,6 +275,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 1.75,
outputPricePerMToken: 14.0,
+ cacheReadPricePerMToken: 0.175,
capabilities: ["function-calling", "streaming"],
},
// GPT-5.4 uses breakpoint pricing above 272K context; we seed
@@ -276,6 +288,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1100000,
inputPricePerMToken: 2.5,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.25,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -284,6 +297,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1100000,
inputPricePerMToken: 0.75,
outputPricePerMToken: 4.5,
+ cacheReadPricePerMToken: 0.075,
capabilities: ["function-calling", "streaming"],
},
{
@@ -292,6 +306,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1100000,
inputPricePerMToken: 0.2,
outputPricePerMToken: 1.25,
+ cacheReadPricePerMToken: 0.02,
capabilities: ["function-calling", "streaming"],
},
{
@@ -300,6 +315,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 400000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 30.0,
+ cacheReadPricePerMToken: 0.5,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -308,6 +324,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 2.0,
outputPricePerMToken: 8.0,
+ cacheReadPricePerMToken: 0.5,
capabilities: ["function-calling", "reasoning", "streaming"],
},
{
@@ -316,6 +333,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 1.1,
outputPricePerMToken: 4.4,
+ cacheReadPricePerMToken: 0.55,
capabilities: ["function-calling", "reasoning", "streaming"],
},
{
@@ -324,6 +342,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 1.1,
outputPricePerMToken: 4.4,
+ cacheReadPricePerMToken: 0.275,
capabilities: ["function-calling", "reasoning", "streaming"],
},
],
@@ -334,6 +353,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -342,6 +363,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -350,6 +373,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 0.8,
outputPricePerMToken: 4.0,
+ cacheReadPricePerMToken: 0.08,
+ cacheCreationPricePerMToken: 1.0,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -358,6 +383,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 0.25,
outputPricePerMToken: 1.25,
+ cacheReadPricePerMToken: 0.03,
+ cacheCreationPricePerMToken: 0.3,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -366,6 +393,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -374,6 +403,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
// Claude 4 family. Cursor/Claude Code stamp the bare id
@@ -389,6 +420,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 1.0,
outputPricePerMToken: 5.0,
+ cacheReadPricePerMToken: 0.1,
+ cacheCreationPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -397,6 +430,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 1.0,
outputPricePerMToken: 5.0,
+ cacheReadPricePerMToken: 0.1,
+ cacheCreationPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -405,6 +440,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -413,6 +450,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -421,6 +460,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -429,6 +470,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 3.0,
outputPricePerMToken: 15.0,
+ cacheReadPricePerMToken: 0.3,
+ cacheCreationPricePerMToken: 3.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -437,6 +480,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 15.0,
outputPricePerMToken: 75.0,
+ cacheReadPricePerMToken: 1.5,
+ cacheCreationPricePerMToken: 18.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -445,6 +490,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 15.0,
outputPricePerMToken: 75.0,
+ cacheReadPricePerMToken: 1.5,
+ cacheCreationPricePerMToken: 18.75,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -453,6 +500,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 200000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -461,6 +510,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -469,6 +520,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
// Cursor thinking-mode SKUs — the model is still Opus 4.7
@@ -481,6 +534,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -489,6 +544,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -497,6 +554,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -505,6 +564,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 5.0,
outputPricePerMToken: 25.0,
+ cacheReadPricePerMToken: 0.5,
+ cacheCreationPricePerMToken: 6.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
],
@@ -515,6 +576,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 2000000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 5.0,
+ cacheReadPricePerMToken: 0.3125,
+ cacheCreationPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -523,6 +586,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.075,
outputPricePerMToken: 0.3,
+ cacheReadPricePerMToken: 0.01875,
+ cacheCreationPricePerMToken: 0.075,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -544,6 +609,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.1,
outputPricePerMToken: 0.4,
+ cacheReadPricePerMToken: 0.01,
+ cacheCreationPricePerMToken: 0.1,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -552,6 +619,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 2000000,
inputPricePerMToken: 1.25,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 0.125,
+ cacheCreationPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -560,6 +629,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.3,
outputPricePerMToken: 2.5,
+ cacheReadPricePerMToken: 0.03,
+ cacheCreationPricePerMToken: 0.3,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -568,6 +639,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.1,
outputPricePerMToken: 0.4,
+ cacheReadPricePerMToken: 0.01,
+ cacheCreationPricePerMToken: 0.1,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -576,6 +649,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.5,
outputPricePerMToken: 3.0,
+ cacheReadPricePerMToken: 0.05,
+ cacheCreationPricePerMToken: 0.5,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
{
@@ -584,6 +659,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 1000000,
inputPricePerMToken: 0.25,
outputPricePerMToken: 1.5,
+ cacheReadPricePerMToken: 0.025,
+ cacheCreationPricePerMToken: 0.25,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -592,6 +669,8 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 2000000,
inputPricePerMToken: 2.0,
outputPricePerMToken: 12.0,
+ cacheReadPricePerMToken: 0.2,
+ cacheCreationPricePerMToken: 2.0,
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
],
@@ -654,6 +733,42 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
outputPricePerMToken: 0.2,
capabilities: ["streaming"],
},
+ {
+ id: "moonshotai/kimi-k2-instruct",
+ displayName: "Kimi K2 Instruct",
+ contextWindow: 128000,
+ inputPricePerMToken: 1.0,
+ outputPricePerMToken: 3.0,
+ cacheReadPricePerMToken: 0.5,
+ capabilities: ["streaming", "function-calling"],
+ },
+ {
+ id: "moonshotai/kimi-k2-instruct-0905",
+ displayName: "Kimi K2 Instruct (0905)",
+ contextWindow: 128000,
+ inputPricePerMToken: 1.0,
+ outputPricePerMToken: 3.0,
+ cacheReadPricePerMToken: 0.5,
+ capabilities: ["streaming", "function-calling"],
+ },
+ {
+ id: "openai/gpt-oss-20b",
+ displayName: "GPT-OSS 20B",
+ contextWindow: 128000,
+ inputPricePerMToken: 0.075,
+ outputPricePerMToken: 0.3,
+ cacheReadPricePerMToken: 0.0375,
+ capabilities: ["streaming"],
+ },
+ {
+ id: "openai/gpt-oss-120b",
+ displayName: "GPT-OSS 120B",
+ contextWindow: 128000,
+ inputPricePerMToken: 0.15,
+ outputPricePerMToken: 0.6,
+ cacheReadPricePerMToken: 0.075,
+ capabilities: ["streaming"],
+ },
],
perplexity: [
{
@@ -688,6 +803,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 128000,
inputPricePerMToken: 2.5,
outputPricePerMToken: 10.0,
+ cacheReadPricePerMToken: 1.25,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -696,6 +812,7 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
contextWindow: 128000,
inputPricePerMToken: 0.15,
outputPricePerMToken: 0.6,
+ cacheReadPricePerMToken: 0.075,
capabilities: ["function-calling", "vision", "streaming"],
},
{
@@ -868,6 +985,33 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
outputPricePerMToken: 3.0,
capabilities: ["streaming", "real-time"],
},
+ {
+ id: "grok-4",
+ displayName: "Grok 4",
+ contextWindow: 256000,
+ inputPricePerMToken: 2.0,
+ outputPricePerMToken: 8.0,
+ cacheReadPricePerMToken: 0.3,
+ capabilities: ["streaming", "real-time", "function-calling"],
+ },
+ {
+ id: "grok-4.3",
+ displayName: "Grok 4.3",
+ contextWindow: 1000000,
+ inputPricePerMToken: 1.25,
+ outputPricePerMToken: 2.5,
+ cacheReadPricePerMToken: 0.2,
+ capabilities: ["streaming", "real-time", "function-calling"],
+ },
+ {
+ id: "grok-4.5",
+ displayName: "Grok 4.5",
+ contextWindow: 500000,
+ inputPricePerMToken: 2.0,
+ outputPricePerMToken: 6.0,
+ cacheReadPricePerMToken: 0.3,
+ capabilities: ["streaming", "real-time", "function-calling"],
+ },
{
id: "grok-vision-beta",
displayName: "Grok Vision Beta",
diff --git a/src/client/src/lib/platform/providers/models-service.ts b/src/client/src/lib/platform/providers/models-service.ts
index 172a3f9f..d58fe657 100644
--- a/src/client/src/lib/platform/providers/models-service.ts
+++ b/src/client/src/lib/platform/providers/models-service.ts
@@ -38,6 +38,8 @@ export async function getCustomModels(
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities,
is_default as isDefault
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
@@ -106,6 +108,9 @@ export async function createCustomModel(
context_window: input.contextWindow || 4096,
input_price_per_m_token: input.inputPricePerMToken || 0,
output_price_per_m_token: input.outputPricePerMToken || 0,
+ cache_read_price_per_m_token: input.cacheReadPricePerMToken || 0,
+ cache_creation_price_per_m_token:
+ input.cacheCreationPricePerMToken || 0,
capabilities: input.capabilities || [],
is_default: false,
created_by_user_id: userId,
@@ -130,6 +135,8 @@ export async function createCustomModel(
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities,
is_default as isDefault
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
@@ -178,6 +185,16 @@ export async function updateCustomModel(
if (input.outputPricePerMToken !== undefined) {
updateFields.push(`output_price_per_m_token = ${input.outputPricePerMToken}`);
}
+ if (input.cacheReadPricePerMToken !== undefined) {
+ updateFields.push(
+ `cache_read_price_per_m_token = ${input.cacheReadPricePerMToken}`
+ );
+ }
+ if (input.cacheCreationPricePerMToken !== undefined) {
+ updateFields.push(
+ `cache_creation_price_per_m_token = ${input.cacheCreationPricePerMToken}`
+ );
+ }
if (input.capabilities !== undefined) {
const capabilitiesArray = input.capabilities
.map((c) => `'${Sanitizer.sanitizeValue(c)}'`)
@@ -260,6 +277,8 @@ export async function getCustomModelsForProvider(
contextWindow: model.contextWindow,
inputPricePerMToken: model.inputPricePerMToken,
outputPricePerMToken: model.outputPricePerMToken,
+ cacheReadPricePerMToken: model.cacheReadPricePerMToken,
+ cacheCreationPricePerMToken: model.cacheCreationPricePerMToken,
capabilities: model.capabilities,
}));
diff --git a/src/client/src/lib/platform/providers/provider-registry.ts b/src/client/src/lib/platform/providers/provider-registry.ts
index 373478a4..3012b73e 100644
--- a/src/client/src/lib/platform/providers/provider-registry.ts
+++ b/src/client/src/lib/platform/providers/provider-registry.ts
@@ -98,6 +98,8 @@ async function loadAllModelsFromDb(
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
ORDER BY provider, is_default DESC, created_at DESC
@@ -118,6 +120,8 @@ async function loadAllModelsFromDb(
contextWindow: Number(row.contextWindow) || 0,
inputPricePerMToken: Number(row.inputPricePerMToken) || 0,
outputPricePerMToken: Number(row.outputPricePerMToken) || 0,
+ cacheReadPricePerMToken: Number(row.cacheReadPricePerMToken) || 0,
+ cacheCreationPricePerMToken: Number(row.cacheCreationPricePerMToken) || 0,
capabilities: row.capabilities || [],
});
}
@@ -136,6 +140,8 @@ async function loadProviderModelsFromDb(
context_window as contextWindow,
input_price_per_m_token as inputPricePerMToken,
output_price_per_m_token as outputPricePerMToken,
+ cache_read_price_per_m_token as cacheReadPricePerMToken,
+ cache_creation_price_per_m_token as cacheCreationPricePerMToken,
capabilities
FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
WHERE provider = '${Sanitizer.sanitizeValue(providerId)}'
@@ -151,6 +157,8 @@ async function loadProviderModelsFromDb(
contextWindow: Number(row.contextWindow) || 0,
inputPricePerMToken: Number(row.inputPricePerMToken) || 0,
outputPricePerMToken: Number(row.outputPricePerMToken) || 0,
+ cacheReadPricePerMToken: Number(row.cacheReadPricePerMToken) || 0,
+ cacheCreationPricePerMToken: Number(row.cacheCreationPricePerMToken) || 0,
capabilities: row.capabilities || [],
}));
}
diff --git a/src/client/src/types/openground.ts b/src/client/src/types/openground.ts
index 8b13866c..410bb04a 100644
--- a/src/client/src/types/openground.ts
+++ b/src/client/src/types/openground.ts
@@ -35,6 +35,10 @@ export interface ModelMetadata {
contextWindow: number;
inputPricePerMToken: number; // Price per million tokens
outputPricePerMToken: number;
+ /** Optional USD per million cache-read tokens (prompt cache hits). */
+ cacheReadPricePerMToken?: number;
+ /** Optional USD per million cache-creation / cache-write tokens. */
+ cacheCreationPricePerMToken?: number;
capabilities?: string[]; // e.g., ["function-calling", "vision", "streaming"]
}
@@ -78,6 +82,8 @@ export interface CustomModelInput {
contextWindow?: number;
inputPricePerMToken?: number;
outputPricePerMToken?: number;
+ cacheReadPricePerMToken?: number;
+ cacheCreationPricePerMToken?: number;
capabilities?: string[];
}