Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: record reasoning output tokens without double counting token usage (#1476)
Roy Tongceb3de22dc34
diff --git a/sdk/python/src/openlit/instrumentation/openai/utils.py b/sdk/python/src/openlit/instrumentation/openai/utils.pyindex 911bcf8f..33edc0ea 100644--- a/sdk/python/src/openlit/instrumentation/openai/utils.py+++ b/sdk/python/src/openlit/instrumentation/openai/utils.py@@ -104,6 +104,19 @@ def extract_reasoning_content(payload):return reasoning if isinstance(reasoning, str) else ""+def extract_reasoning_tokens(usage, details_key):+ """Return reasoning tokens (a subset of output tokens) from a usage dict.++ Works for both chat completions (``completion_tokens_details``) and the+ responses API (``output_tokens_details``) and falls back to 0 whenever the+ details object is missing or not a dict.+ """+ details = (usage or {}).get(details_key) or {}+ if not isinstance(details, dict):+ details = {}+ return details.get("reasoning_tokens", 0) or 0++def format_content(messages):"""Format the messages into a string for span events.@@ -750,6 +763,11 @@ def process_chat_chunk(scope, chunk):scope._cache_creation_input_tokens = (prompt_tokens_details.get("cache_write_tokens", 0) or 0)+ # Reasoning tokens (subset of completion_tokens) from the final+ # streaming chunk (stream_options={"include_usage": True}).+ scope._reasoning_tokens = extract_reasoning_tokens(+ usage, "completion_tokens_details"+ )scope._system_fingerprint = (chunked.get("system_fingerprint") or scope._system_fingerprint@@ -855,8 +873,9 @@ def process_response_chunk(scope, chunk):scope._output_tokens = usage.get("output_tokens", 0)# Handle reasoning tokens- output_tokens_details = usage.get("output_tokens_details", {})- scope._reasoning_tokens = output_tokens_details.get("reasoning_tokens", 0)+ scope._reasoning_tokens = extract_reasoning_tokens(+ usage, "output_tokens_details"+ )# Cached tokens (OTel: gen_ai.usage.cache_read.input_tokens)# Cache writes (OTel: gen_ai.usage.cache_creation.input_tokens)@@ -1055,8 +1074,15 @@ def common_response_logic()scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)- # Reasoning tokens+ # Reasoning tokens. OTel: gen_ai.usage.reasoning.output_tokens is a subset+ # of gen_ai.usage.output_tokens (already set above), so it is recorded as a+ # separate attribute and never added on top of the output total.if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:+ scope._span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,+ scope._reasoning_tokens,+ )+ # OpenLIT legacy alias (pre-OTel naming), kept for backward compat.scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,scope._reasoning_tokens,@@ -1334,8 +1360,9 @@ def process_response_response(scope._input_tokens = usage.get("input_tokens", 0)scope._output_tokens = usage.get("output_tokens", 0)- output_tokens_details = usage.get("output_tokens_details", {})- scope._reasoning_tokens = output_tokens_details.get("reasoning_tokens", 0)+ scope._reasoning_tokens = extract_reasoning_tokens(+ usage, "output_tokens_details"+ )input_tokens_details = usage.get("input_tokens_details", {}) or {}if not isinstance(input_tokens_details, dict):@@ -1562,6 +1589,20 @@ def common_chat_logic()scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)+ # Reasoning tokens (OTel: gen_ai.usage.reasoning.output_tokens is a subset+ # of gen_ai.usage.output_tokens above, so it is recorded separately and+ # never added on top of the output total / token-usage metric).+ if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:+ scope._span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,+ scope._reasoning_tokens,+ )+ # OpenLIT legacy alias (pre-OTel naming), kept for backward compat.+ scope._span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,+ scope._reasoning_tokens,+ )+# OTel cached token attributes (set even when 0)if hasattr(scope, "_cache_read_input_tokens"):scope._span.set_attribute(@@ -1783,6 +1824,13 @@ def process_chat_response(scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)scope._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)+ # OpenAI chat completions report reasoning tokens under+ # usage.completion_tokens_details.reasoning_tokens (o1/o3 family). These are+ # a subset of completion_tokens, which already includes them.+ scope._reasoning_tokens = extract_reasoning_tokens(+ response_dict.get("usage", {}), "completion_tokens_details"+ )+# Extract cache tokens (OpenAI prompt caching)prompt_tokens_details = (response_dict.get("usage", {}).get("prompt_tokens_details", {}) or {}diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 6b290659..09d22a7c 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -571,6 +571,9 @@ class SemanticConvention:GEN_AI_USAGE_COMPLETION_TOKENS_DETAILS_REASONING = ("gen_ai.usage.completion_tokens_details.reasoning_tokens")+ # OTel GenAI semconv (experimental): reasoning output tokens are a subset of+ # gen_ai.usage.output_tokens and MUST NOT be added on top of it.+ GEN_AI_USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning.output_tokens"GEN_AI_USAGE_REASONING_TOKENS = "gen_ai.usage.reasoning_tokens"GEN_AI_USAGE_PROMPT_TOKENS_DETAILS_CACHE_READ = ("gen_ai.usage.prompt_tokens_details.cached_tokens"diff --git a/sdk/python/tests/test_openai_reasoning_tokens.py b/sdk/python/tests/test_openai_reasoning_tokens.pynew file mode 100644index 00000000..8c3e2f58--- /dev/null+++ b/sdk/python/tests/test_openai_reasoning_tokens.py@@ -0,0 +1,440 @@+"""Invariant tests for reasoning output token telemetry (Issue #1474).++OpenAI reports reasoning tokens as a *subset* of the completion/output tokens:+a response with ``completion_tokens=1000`` /+``completion_tokens_details.reasoning_tokens=700`` (chat completions) or+``output_tokens=1000`` / ``output_tokens_details.reasoning_tokens=700``+(responses API) uses 700 tokens for reasoning, and those 700 are already+included in the 1000 output tokens.++Per OTel GenAI semantic conventions:+ * ``gen_ai.usage.reasoning.output_tokens`` = 700 (the subset)+ * ``gen_ai.usage.output_tokens`` = 1000 (already includes the subset)+ * ``gen_ai.client.token.usage{gen_ai.token.type=output}`` = 1000 (must NOT+ become 1700 -- reasoning is never added on top of the output total).++These tests lock in that invariant for the Python OpenAI instrumentation.+"""++import time+from types import SimpleNamespace+from unittest.mock import MagicMock++from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.openai.utils import (+ process_chat_chunk,+ process_chat_response,+ process_response_response,+ process_streaming_chat_response,+)+from openlit.semcov import SemanticConvention+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ return tracer_provider.get_tracer("test-openai-reasoning-tokens"), exporter+++def _metrics_dict():+ """MagicMock metric instruments mirroring openlit.otel.metrics.build_metrics()."""+ return {+ "genai_client_usage_tokens": MagicMock(),+ "genai_client_operation_duration": MagicMock(),+ "genai_client_time_to_first_chunk": MagicMock(),+ "genai_client_time_per_output_chunk": MagicMock(),+ "genai_server_tbt": MagicMock(),+ "genai_server_ttft": MagicMock(),+ "genai_server_request_duration": MagicMock(),+ "genai_cost": MagicMock(),+ }+++def _output_token_usage_records(metrics):+ """Return [(value, attrs)] of gen_ai.client.token.usage{token_type=output}."""+ records = []+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:+ value, attrs = call.args+ if (+ attrs.get(SemanticConvention.GEN_AI_TOKEN_TYPE)+ == SemanticConvention.GEN_AI_TOKEN_TYPE_OUTPUT+ ):+ records.append((value, attrs))+ return records+++def _assert_no_double_counting(attrs, metrics, output_tokens=1000, reasoning_tokens=700):+ """Shared invariant: reasoning is a subset of output, never added on top."""+ # Span attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == output_tokens+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]+ == reasoning_tokens+ )+ # OpenLIT legacy alias (pre-OTel naming) must stay populated for backward+ # compatibility.+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == reasoning_tokens++ # gen_ai.client.token.usage{token_type=output} must stay 1000, not 1700+ output_records = _output_token_usage_records(metrics)+ assert output_records, "expected at least one output token usage record"+ for value, _attrs in output_records:+ assert value == output_tokens+ assert (+ _attrs.get(SemanticConvention.GEN_AI_TOKEN_TYPE)+ == SemanticConvention.GEN_AI_TOKEN_TYPE_OUTPUT+ )+ assert all(+ value != output_tokens + reasoning_tokens for value, _ in output_records+ ), "reasoning tokens must not be added on top of the output total"+++def _stream_scope(span):+ return SimpleNamespace(+ _span=span,+ _llmresponse="",+ _response_id="",+ _response_model="",+ _finish_reason="",+ _system_fingerprint="",+ _service_tier="auto",+ _tools=None,+ _kwargs={+ "model": "o3-mini",+ "messages": [{"role": "user", "content": "think step by step"}],+ },+ _start_time=time.time(),+ _end_time=None,+ _timestamps=[],+ _ttft=0,+ _tbt=0,+ _server_address="api.openai.com",+ _server_port=443,+ )+++def test_chat_completions_reasoning_tokens_are_subset_of_output():+ """o1/o3-style chat completion: reasoning stays a subset of output tokens."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ response = {+ "id": "chatcmpl_o1",+ "model": "o3-mini",+ "choices": [+ {+ "message": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": "stop",+ }+ ],+ "usage": {+ "prompt_tokens": 500,+ "completion_tokens": 1000,+ "completion_tokens_details": {"reasoning_tokens": 700, "text_tokens": 300},+ },+ }++ with tracer.start_as_current_span("chat o3-mini") as span:+ process_chat_response(+ response,+ request_model="o3-mini",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="o3-mini",+ messages=[{"role": "user", "content": "think step by step"}],+ )++ _assert_no_double_counting(exporter.get_finished_spans()[0].attributes, metrics)+++def test_streaming_chat_completions_reasoning_tokens_are_subset_of_output():+ """Streaming chat completion with include_usage: reasoning is a subset."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ span = tracer.start_span("chat o3-mini")+ scope = _stream_scope(span)++ process_chat_chunk(+ scope,+ {+ "id": "chatcmpl_o1",+ "model": "o3-mini",+ "choices": [{"delta": {"content": "The answer is 42."}}],+ },+ )+ # Final chunk carries usage (stream_options={"include_usage": True}).+ process_chat_chunk(+ scope,+ {+ "choices": [],+ "usage": {+ "prompt_tokens": 500,+ "completion_tokens": 1000,+ "completion_tokens_details": {+ "reasoning_tokens": 700,+ "text_tokens": 300,+ },+ },+ },+ )++ with span:+ process_streaming_chat_response(+ scope,+ pricing_info={},+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ )++ _assert_no_double_counting(exporter.get_finished_spans()[0].attributes, metrics)+++def test_responses_api_reasoning_tokens_are_subset_of_output():+ """Responses API: output_tokens_details.reasoning_tokens is a subset."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ response = {+ "id": "resp_o1",+ "model": "o3-mini",+ "status": "completed",+ "output": [+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "The answer is 42."}],+ }+ ],+ "usage": {+ "input_tokens": 500,+ "output_tokens": 1000,+ "output_tokens_details": {"reasoning_tokens": 700},+ },+ }++ with tracer.start_as_current_span("responses o3-mini") as span:+ process_response_response(+ response,+ request_model="o3-mini",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="o3-mini",+ input="think step by step",+ )++ _assert_no_double_counting(exporter.get_finished_spans()[0].attributes, metrics)+++def test_no_reasoning_tokens_omits_reasoning_attribute():+ """No completion_tokens_details: reasoning attribute absent, output intact."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ response = {+ "id": "chatcmpl_plain",+ "model": "gpt-4o",+ "choices": [+ {+ "message": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 500, "completion_tokens": 1000},+ }++ with tracer.start_as_current_span("chat gpt-4o") as span:+ process_chat_response(+ response,+ request_model="gpt-4o",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="gpt-4o",+ messages=[{"role": "user", "content": "hi"}],+ )++ attrs = exporter.get_finished_spans()[0].attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs++ output_records = _output_token_usage_records(metrics)+ assert output_records+ for value, _attrs in output_records:+ assert value == 1000+++def test_zero_reasoning_tokens_omits_reasoning_attribute():+ """reasoning_tokens present but 0: no reasoning attrs, output intact."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ response = {+ "id": "chatcmpl_zero",+ "model": "gpt-4o",+ "choices": [+ {+ "message": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": "stop",+ }+ ],+ "usage": {+ "prompt_tokens": 500,+ "completion_tokens": 1000,+ "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 1000},+ },+ }++ with tracer.start_as_current_span("chat gpt-4o zero") as span:+ process_chat_response(+ response,+ request_model="gpt-4o",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="gpt-4o",+ messages=[{"role": "user", "content": "hi"}],+ )++ attrs = exporter.get_finished_spans()[0].attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs++ for value, _attrs in _output_token_usage_records(metrics):+ assert value == 1000+++def _non_dict_details_chat_response():+ """chat completion whose completion_tokens_details is not a dict."""+ return {+ "id": "chatcmpl_baddetails",+ "model": "gpt-4o",+ "choices": [+ {+ "message": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": "stop",+ }+ ],+ "usage": {+ "prompt_tokens": 500,+ "completion_tokens": 1000,+ "completion_tokens_details": "unavailable",+ },+ }+++def test_non_dict_completion_tokens_details_does_not_crash():+ """Defensive casting: non-dict details fall back to {}, no exception."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()++ with tracer.start_as_current_span("chat bad details") as span:+ process_chat_response(+ _non_dict_details_chat_response(),+ request_model="gpt-4o",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="gpt-4o",+ messages=[{"role": "user", "content": "hi"}],+ )++ attrs = exporter.get_finished_spans()[0].attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+++def test_non_dict_output_tokens_details_does_not_crash():+ """Responses API: non-dict output_tokens_details falls back to {}."""+ tracer, exporter = _tracer_and_exporter()+ metrics = _metrics_dict()+ response = {+ "id": "resp_baddetails",+ "model": "o3-mini",+ "status": "completed",+ "output": [+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "The answer is 42."}],+ }+ ],+ "usage": {+ "input_tokens": 500,+ "output_tokens": 1000,+ "output_tokens_details": None,+ },+ }++ with tracer.start_as_current_span("responses bad details") as span:+ process_response_response(+ response,+ request_model="o3-mini",+ pricing_info={},+ server_port=443,+ server_address="api.openai.com",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ model="o3-mini",+ input="think step by step",+ )++ attrs = exporter.get_finished_spans()[0].attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrsdiff --git a/sdk/python/tests/test_tool_call_list_attrs.py b/sdk/python/tests/test_tool_call_list_attrs.pyindex 6ff8f548..a7886265 100644--- a/sdk/python/tests/test_tool_call_list_attrs.py+++ b/sdk/python/tests/test_tool_call_list_attrs.py@@ -50,19 +50,19 @@ def _tracer_with_exporter():return provider.get_tracer(__name__), exporter-_COMMON_KWARGS = dict(- request_model="test-model",- pricing_info={},- server_port=443,- server_address="api.example.com",- environment="test",- application_name="regression-test",- metrics=None,- start_time=time.time(),- capture_message_content=False,- disable_metrics=True,- version="1.0.0",-)+_COMMON_KWARGS = {+ "request_model": "test-model",+ "pricing_info": {},+ "server_port": 443,+ "server_address": "api.example.com",+ "environment": "test",+ "application_name": "regression-test",+ "metrics": None,+ "start_time": time.time(),+ "capture_message_content": False,+ "disable_metrics": True,+ "version": "1.0.0",+}# ---------------------------------------------------------------------------