Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(cost): price cached tokens at cache rates in litellm and claude_agent_sdk (#1379)

Ehsane59fcff6f72b
diff --git a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
index 6a8c97da..5a4ff9ca 100644
--- a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
+++ b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py
@@ -321,7 +321,20 @@ def set_chat_span_attributes(
output_tokens = usage_attrs.get(
SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 0
)
- cost = _calculate_cost(model_str, pricing_info, input_tokens, output_tokens)
+ cache_read_tokens = usage_attrs.get(
+ SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, 0
+ )
+ cache_creation_tokens = usage_attrs.get(
+ SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
+ )
+ cost = _calculate_cost(
+ model_str,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens,
+ cache_creation_tokens,
+ )
span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
output_messages = None
@@ -427,11 +440,34 @@ def _build_output_messages(message, mapped_finish_reason):
return None
-def _calculate_cost(model_str, pricing_info, input_tokens, output_tokens):
- """Estimate cost using the shared OpenLIT pricing helper."""
+def _calculate_cost(
+ model_str,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+):
+ """Estimate cost using the shared OpenLIT pricing helper.
+
+ ``input_tokens`` is the inclusive total built by ``extract_usage``
+ (uncached input + cache-read + cache-creation), so the cache token counts
+ are forwarded with ``prompt_tokens_include_cache=True``. This re-prices the
+ cached tokens at their dedicated cache rates instead of the full prompt
+ rate; without it, cache-read tokens (billed at 1/10th of prompt price for
+ Claude models) are charged at full prompt price.
+ """
if not pricing_info or not model_str:
return 0
- return get_chat_model_cost(model_str, pricing_info, input_tokens, output_tokens)
+ return get_chat_model_cost(
+ model_str,
+ pricing_info,
+ input_tokens,
+ output_tokens,
+ cache_read_tokens=cache_read_tokens,
+ cache_creation_tokens=cache_creation_tokens,
+ prompt_tokens_include_cache=True,
+ )
def _emit_chat_inference_event(
diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.py
index 5c52727c..ffc08bad 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py
@@ -328,6 +328,23 @@ def emit_inference_event(
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
+def _extract_cache_tokens(usage):
+ """Return ``(cache_read, cache_creation)`` from a litellm usage payload.
+
+ litellm normalizes both cache counts into ``prompt_tokens_details``: the
+ OpenAI-style ``cached_tokens`` is the cache-read count and, for Anthropic,
+ ``cache_creation_tokens`` is the cache-write count. The cache-creation count
+ was previously read from ``completion_tokens_details.cached_tokens``, a
+ field litellm never populates, so it was always 0.
+ """
+ prompt_tokens_details = (
+ usage.get("prompt_tokens_details", usage.get("input_tokens_details", {})) or {}
+ )
+ cache_read = prompt_tokens_details.get("cached_tokens", 0) or 0
+ cache_creation = prompt_tokens_details.get("cache_creation_tokens", 0) or 0
+ return cache_read, cache_creation
+
+
def process_chunk(scope, chunk):
"""
Process a chunk of response data and update state.
@@ -400,14 +417,10 @@ def process_chunk(scope, chunk):
usage = chunked.get("usage", {})
scope._input_tokens = usage.get("prompt_tokens", 0)
scope._output_tokens = usage.get("completion_tokens", 0)
- prompt_tokens_details = (
- usage.get("prompt_tokens_details", usage.get("input_tokens_details", {}))
- or {}
- )
- scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
- scope._cache_creation_input_tokens = (
- usage.get("completion_tokens_details") or {}
- ).get("cached_tokens", 0)
+ (
+ scope._cache_read_input_tokens,
+ scope._cache_creation_input_tokens,
+ ) = _extract_cache_tokens(usage)
scope._end_time = time.time()
@@ -433,8 +446,18 @@ def common_chat_logic(
request_model = scope._kwargs.get("model", "openai/gpt-4o")
+ # litellm normalizes ``prompt_tokens`` to include cache-read and
+ # cache-creation tokens (OpenAI convention), so forward the cache counts
+ # with ``prompt_tokens_include_cache=True`` to re-price them at their cache
+ # rates instead of the full prompt rate.
cost = get_chat_model_cost(
- request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ request_model,
+ pricing_info,
+ scope._input_tokens,
+ scope._output_tokens,
+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0),
+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0),
+ prompt_tokens_include_cache=True,
)
# Common Span Attributes
@@ -761,13 +784,10 @@ def process_chat_response(
# Handle token usage including reasoning tokens and cached tokens
scope._input_tokens = usage.get("prompt_tokens", 0)
scope._output_tokens = usage.get("completion_tokens", 0)
- prompt_tokens_details = (
- usage.get("prompt_tokens_details", usage.get("input_tokens_details", {})) or {}
- )
- scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
- scope._cache_creation_input_tokens = (
- usage.get("completion_tokens_details") or {}
- ).get("cached_tokens", 0)
+ (
+ scope._cache_read_input_tokens,
+ scope._cache_creation_input_tokens,
+ ) = _extract_cache_tokens(usage)
scope._response_id = response_dict.get("id") or ""
scope._response_model = response_dict.get("model") or ""
choices = response_dict.get("choices", [])
diff --git a/sdk/python/tests/test_cache_pricing.py b/sdk/python/tests/test_cache_pricing.py
index 2873f31d..f4fad162 100644
--- a/sdk/python/tests/test_cache_pricing.py
+++ b/sdk/python/tests/test_cache_pricing.py
@@ -210,3 +210,127 @@ class TestBundledPricingFile:
# cache read should be cheaper than prompt; cache write pricier.
assert entry["cacheReadPrice"] < entry["promptPrice"]
assert entry["cacheCreationPrice"] > entry["promptPrice"]
+
+
+# 200 uncached input + 5000 cache-read + 1000 cache-creation + 300 output.
+_RAW_INPUT, _CACHE_READ, _CACHE_CREATION, _OUTPUT = 200, 5000, 1000, 300
+_INCLUSIVE_INPUT = _RAW_INPUT + _CACHE_READ + _CACHE_CREATION # 6200
+
+
+def _ground_truth():
+ return (
+ (_RAW_INPUT / 1000) * 0.003
+ + (_OUTPUT / 1000) * 0.015
+ + (_CACHE_READ / 1000) * 0.0003
+ + (_CACHE_CREATION / 1000) * 0.00375
+ )
+
+
+class TestClaudeAgentSDKAdapter:
+ """The claude_agent_sdk adapter builds an inclusive input total in
+ ``extract_usage`` (uncached + cache-read + cache-creation) and must re-price
+ the cache tokens at their cache rates instead of the full prompt rate."""
+
+ USAGE = {
+ "input_tokens": _RAW_INPUT,
+ "cache_read_input_tokens": _CACHE_READ,
+ "cache_creation_input_tokens": _CACHE_CREATION,
+ "output_tokens": _OUTPUT,
+ }
+
+ def test_extract_usage_total_is_inclusive(self):
+ from openlit.instrumentation.claude_agent_sdk.utils import extract_usage
+ from openlit.semcov import SemanticConvention as SC
+
+ attrs = extract_usage(self.USAGE)
+ assert attrs[SC.GEN_AI_USAGE_INPUT_TOKENS] == _INCLUSIVE_INPUT
+ assert attrs[SC.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == _CACHE_READ
+ assert attrs[SC.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] == _CACHE_CREATION
+
+ def test_cost_reprices_cache_tokens(self):
+ from openlit.instrumentation.claude_agent_sdk.utils import (
+ extract_usage,
+ _calculate_cost,
+ )
+ from openlit.semcov import SemanticConvention as SC
+
+ attrs = extract_usage(self.USAGE)
+ cost = _calculate_cost(
+ MODEL,
+ PRICING_WITH_CACHE,
+ attrs[SC.GEN_AI_USAGE_INPUT_TOKENS],
+ attrs[SC.GEN_AI_USAGE_OUTPUT_TOKENS],
+ attrs[SC.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS],
+ attrs[SC.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS],
+ )
+ assert cost == _approx(_ground_truth())
+
+ def test_billing_inclusive_total_at_prompt_price_overcharges(self):
+ # Documents the pre-fix behaviour: billing the inclusive total at prompt
+ # price charges cache-read tokens ~10x their real rate.
+ overcharged = get_chat_model_cost(
+ MODEL, PRICING_WITH_CACHE, _INCLUSIVE_INPUT, _OUTPUT
+ )
+ assert overcharged == _approx(
+ (_INCLUSIVE_INPUT / 1000) * 0.003 + (_OUTPUT / 1000) * 0.015
+ )
+ assert overcharged > _ground_truth()
+
+
+class TestLiteLLMAdapter:
+ """litellm normalizes ``prompt_tokens`` to include cache tokens and reports
+ both cache counts under ``prompt_tokens_details``. The adapter must read
+ cache-creation from there (not the never-populated
+ ``completion_tokens_details.cached_tokens``) and forward the cache counts to
+ the cost helper."""
+
+ # Shape of ``response["usage"]`` as litellm emits it (litellm 1.92.0).
+ USAGE = {
+ "prompt_tokens": _INCLUSIVE_INPUT,
+ "completion_tokens": _OUTPUT,
+ "prompt_tokens_details": {
+ "cached_tokens": _CACHE_READ,
+ "cache_creation_tokens": _CACHE_CREATION,
+ "text_tokens": _RAW_INPUT,
+ },
+ # litellm never sets a cache count here; the old code read it anyway.
+ "completion_tokens_details": {"text_tokens": _OUTPUT, "reasoning_tokens": 0},
+ }
+
+ def test_extract_cache_tokens_reads_prompt_tokens_details(self):
+ from openlit.instrumentation.litellm.utils import _extract_cache_tokens
+
+ cache_read, cache_creation = _extract_cache_tokens(self.USAGE)
+ assert cache_read == _CACHE_READ
+ # Was always 0 when read from completion_tokens_details.cached_tokens.
+ assert cache_creation == _CACHE_CREATION
+
+ def test_dead_field_source_yields_zero(self):
+ # Confirms why the old source was wrong: litellm puts no cache count in
+ # completion_tokens_details, so the previous read was always 0.
+ assert (self.USAGE.get("completion_tokens_details") or {}).get(
+ "cached_tokens", 0
+ ) == 0
+
+ def test_cost_prices_cache_tokens_at_cache_rate(self):
+ cache_read, cache_creation = (_CACHE_READ, _CACHE_CREATION)
+ cost = get_chat_model_cost(
+ MODEL,
+ PRICING_WITH_CACHE,
+ self.USAGE["prompt_tokens"],
+ self.USAGE["completion_tokens"],
+ cache_read_tokens=cache_read,
+ cache_creation_tokens=cache_creation,
+ prompt_tokens_include_cache=True,
+ )
+ assert cost == _approx(_ground_truth())
+
+ def test_missing_cache_forward_overcharges(self):
+ # Pre-fix: the whole inclusive prompt total is billed at prompt price.
+ buggy = get_chat_model_cost(
+ MODEL,
+ PRICING_WITH_CACHE,
+ self.USAGE["prompt_tokens"],
+ self.USAGE["completion_tokens"],
+ )
+ assert buggy > _ground_truth()