Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(cost): price cached tokens at cache rates in litellm and claude_agent_sdk (#1379)
Ehsane59fcff6f72b
diff --git a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.pyindex 6a8c97da..5a4ff9ca 100644--- a/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py+++ b/sdk/python/src/openlit/instrumentation/claude_agent_sdk/utils.py@@ -321,7 +321,20 @@ def set_chat_span_attributes(output_tokens = usage_attrs.get(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 0)- cost = _calculate_cost(model_str, pricing_info, input_tokens, output_tokens)+ cache_read_tokens = usage_attrs.get(+ SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, 0+ )+ cache_creation_tokens = usage_attrs.get(+ SemanticConvention.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0+ )+ cost = _calculate_cost(+ model_str,+ pricing_info,+ input_tokens,+ output_tokens,+ cache_read_tokens,+ cache_creation_tokens,+ )span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)output_messages = None@@ -427,11 +440,34 @@ def _build_output_messages(message, mapped_finish_reason):return None-def _calculate_cost(model_str, pricing_info, input_tokens, output_tokens):- """Estimate cost using the shared OpenLIT pricing helper."""+def _calculate_cost(+ model_str,+ pricing_info,+ input_tokens,+ output_tokens,+ cache_read_tokens=0,+ cache_creation_tokens=0,+):+ """Estimate cost using the shared OpenLIT pricing helper.++ ``input_tokens`` is the inclusive total built by ``extract_usage``+ (uncached input + cache-read + cache-creation), so the cache token counts+ are forwarded with ``prompt_tokens_include_cache=True``. This re-prices the+ cached tokens at their dedicated cache rates instead of the full prompt+ rate; without it, cache-read tokens (billed at 1/10th of prompt price for+ Claude models) are charged at full prompt price.+ """if not pricing_info or not model_str:return 0- return get_chat_model_cost(model_str, pricing_info, input_tokens, output_tokens)+ return get_chat_model_cost(+ model_str,+ pricing_info,+ input_tokens,+ output_tokens,+ cache_read_tokens=cache_read_tokens,+ cache_creation_tokens=cache_creation_tokens,+ prompt_tokens_include_cache=True,+ )def _emit_chat_inference_event(diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.pyindex 5c52727c..ffc08bad 100644--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py@@ -328,6 +328,23 @@ def emit_inference_event(logger.warning("Failed to emit inference event: %s", e, exc_info=True)+def _extract_cache_tokens(usage):+ """Return ``(cache_read, cache_creation)`` from a litellm usage payload.++ litellm normalizes both cache counts into ``prompt_tokens_details``: the+ OpenAI-style ``cached_tokens`` is the cache-read count and, for Anthropic,+ ``cache_creation_tokens`` is the cache-write count. The cache-creation count+ was previously read from ``completion_tokens_details.cached_tokens``, a+ field litellm never populates, so it was always 0.+ """+ prompt_tokens_details = (+ usage.get("prompt_tokens_details", usage.get("input_tokens_details", {})) or {}+ )+ cache_read = prompt_tokens_details.get("cached_tokens", 0) or 0+ cache_creation = prompt_tokens_details.get("cache_creation_tokens", 0) or 0+ return cache_read, cache_creation++def process_chunk(scope, chunk):"""Process a chunk of response data and update state.@@ -400,14 +417,10 @@ def process_chunk(scope, chunk):usage = chunked.get("usage", {})scope._input_tokens = usage.get("prompt_tokens", 0)scope._output_tokens = usage.get("completion_tokens", 0)- prompt_tokens_details = (- usage.get("prompt_tokens_details", usage.get("input_tokens_details", {}))- or {}- )- scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)- scope._cache_creation_input_tokens = (- usage.get("completion_tokens_details") or {}- ).get("cached_tokens", 0)+ (+ scope._cache_read_input_tokens,+ scope._cache_creation_input_tokens,+ ) = _extract_cache_tokens(usage)scope._end_time = time.time()@@ -433,8 +446,18 @@ def common_chat_logic(request_model = scope._kwargs.get("model", "openai/gpt-4o")+ # litellm normalizes ``prompt_tokens`` to include cache-read and+ # cache-creation tokens (OpenAI convention), so forward the cache counts+ # with ``prompt_tokens_include_cache=True`` to re-price them at their cache+ # rates instead of the full prompt rate.cost = get_chat_model_cost(- request_model, pricing_info, scope._input_tokens, scope._output_tokens+ request_model,+ pricing_info,+ scope._input_tokens,+ scope._output_tokens,+ cache_read_tokens=getattr(scope, "_cache_read_input_tokens", 0),+ cache_creation_tokens=getattr(scope, "_cache_creation_input_tokens", 0),+ prompt_tokens_include_cache=True,)# Common Span Attributes@@ -761,13 +784,10 @@ def process_chat_response(# Handle token usage including reasoning tokens and cached tokensscope._input_tokens = usage.get("prompt_tokens", 0)scope._output_tokens = usage.get("completion_tokens", 0)- prompt_tokens_details = (- usage.get("prompt_tokens_details", usage.get("input_tokens_details", {})) or {}- )- scope._cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)- scope._cache_creation_input_tokens = (- usage.get("completion_tokens_details") or {}- ).get("cached_tokens", 0)+ (+ scope._cache_read_input_tokens,+ scope._cache_creation_input_tokens,+ ) = _extract_cache_tokens(usage)scope._response_id = response_dict.get("id") or ""scope._response_model = response_dict.get("model") or ""choices = response_dict.get("choices", [])diff --git a/sdk/python/tests/test_cache_pricing.py b/sdk/python/tests/test_cache_pricing.pyindex 2873f31d..f4fad162 100644--- a/sdk/python/tests/test_cache_pricing.py+++ b/sdk/python/tests/test_cache_pricing.py@@ -210,3 +210,127 @@ class TestBundledPricingFile:# cache read should be cheaper than prompt; cache write pricier.assert entry["cacheReadPrice"] < entry["promptPrice"]assert entry["cacheCreationPrice"] > entry["promptPrice"]+++# 200 uncached input + 5000 cache-read + 1000 cache-creation + 300 output.+_RAW_INPUT, _CACHE_READ, _CACHE_CREATION, _OUTPUT = 200, 5000, 1000, 300+_INCLUSIVE_INPUT = _RAW_INPUT + _CACHE_READ + _CACHE_CREATION # 6200+++def _ground_truth():+ return (+ (_RAW_INPUT / 1000) * 0.003+ + (_OUTPUT / 1000) * 0.015+ + (_CACHE_READ / 1000) * 0.0003+ + (_CACHE_CREATION / 1000) * 0.00375+ )+++class TestClaudeAgentSDKAdapter:+ """The claude_agent_sdk adapter builds an inclusive input total in+ ``extract_usage`` (uncached + cache-read + cache-creation) and must re-price+ the cache tokens at their cache rates instead of the full prompt rate."""++ USAGE = {+ "input_tokens": _RAW_INPUT,+ "cache_read_input_tokens": _CACHE_READ,+ "cache_creation_input_tokens": _CACHE_CREATION,+ "output_tokens": _OUTPUT,+ }++ def test_extract_usage_total_is_inclusive(self):+ from openlit.instrumentation.claude_agent_sdk.utils import extract_usage+ from openlit.semcov import SemanticConvention as SC++ attrs = extract_usage(self.USAGE)+ assert attrs[SC.GEN_AI_USAGE_INPUT_TOKENS] == _INCLUSIVE_INPUT+ assert attrs[SC.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == _CACHE_READ+ assert attrs[SC.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] == _CACHE_CREATION++ def test_cost_reprices_cache_tokens(self):+ from openlit.instrumentation.claude_agent_sdk.utils import (+ extract_usage,+ _calculate_cost,+ )+ from openlit.semcov import SemanticConvention as SC++ attrs = extract_usage(self.USAGE)+ cost = _calculate_cost(+ MODEL,+ PRICING_WITH_CACHE,+ attrs[SC.GEN_AI_USAGE_INPUT_TOKENS],+ attrs[SC.GEN_AI_USAGE_OUTPUT_TOKENS],+ attrs[SC.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS],+ attrs[SC.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS],+ )+ assert cost == _approx(_ground_truth())++ def test_billing_inclusive_total_at_prompt_price_overcharges(self):+ # Documents the pre-fix behaviour: billing the inclusive total at prompt+ # price charges cache-read tokens ~10x their real rate.+ overcharged = get_chat_model_cost(+ MODEL, PRICING_WITH_CACHE, _INCLUSIVE_INPUT, _OUTPUT+ )+ assert overcharged == _approx(+ (_INCLUSIVE_INPUT / 1000) * 0.003 + (_OUTPUT / 1000) * 0.015+ )+ assert overcharged > _ground_truth()+++class TestLiteLLMAdapter:+ """litellm normalizes ``prompt_tokens`` to include cache tokens and reports+ both cache counts under ``prompt_tokens_details``. The adapter must read+ cache-creation from there (not the never-populated+ ``completion_tokens_details.cached_tokens``) and forward the cache counts to+ the cost helper."""++ # Shape of ``response["usage"]`` as litellm emits it (litellm 1.92.0).+ USAGE = {+ "prompt_tokens": _INCLUSIVE_INPUT,+ "completion_tokens": _OUTPUT,+ "prompt_tokens_details": {+ "cached_tokens": _CACHE_READ,+ "cache_creation_tokens": _CACHE_CREATION,+ "text_tokens": _RAW_INPUT,+ },+ # litellm never sets a cache count here; the old code read it anyway.+ "completion_tokens_details": {"text_tokens": _OUTPUT, "reasoning_tokens": 0},+ }++ def test_extract_cache_tokens_reads_prompt_tokens_details(self):+ from openlit.instrumentation.litellm.utils import _extract_cache_tokens++ cache_read, cache_creation = _extract_cache_tokens(self.USAGE)+ assert cache_read == _CACHE_READ+ # Was always 0 when read from completion_tokens_details.cached_tokens.+ assert cache_creation == _CACHE_CREATION++ def test_dead_field_source_yields_zero(self):+ # Confirms why the old source was wrong: litellm puts no cache count in+ # completion_tokens_details, so the previous read was always 0.+ assert (self.USAGE.get("completion_tokens_details") or {}).get(+ "cached_tokens", 0+ ) == 0++ def test_cost_prices_cache_tokens_at_cache_rate(self):+ cache_read, cache_creation = (_CACHE_READ, _CACHE_CREATION)+ cost = get_chat_model_cost(+ MODEL,+ PRICING_WITH_CACHE,+ self.USAGE["prompt_tokens"],+ self.USAGE["completion_tokens"],+ cache_read_tokens=cache_read,+ cache_creation_tokens=cache_creation,+ prompt_tokens_include_cache=True,+ )+ assert cost == _approx(_ground_truth())++ def test_missing_cache_forward_overcharges(self):+ # Pre-fix: the whole inclusive prompt total is billed at prompt price.+ buggy = get_chat_model_cost(+ MODEL,+ PRICING_WITH_CACHE,+ self.USAGE["prompt_tokens"],+ self.USAGE["completion_tokens"],+ )+ assert buggy > _ground_truth()