Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(azure_ai_inference): keep reasoning tokens out of gen_ai.client.token.usage (#1595)

Dmitry Voropaev42e11e4a115a
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index 2b1ac2ee..e4ee415e 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -238,6 +238,70 @@ def set_span_reasoning_content(span, scope, capture_message_content=True):
)
+def extract_reasoning_tokens(usage, details_key):
+ """Return reasoning tokens (a subset of output tokens) from a usage dict,
+ or None when the provider did not report the details object.
+
+ Works for both chat completions (``completion_tokens_details``) and the
+ responses API (``output_tokens_details``). A missing or non-dict details
+ object means "unknown" and stays distinguishable from an explicit
+ ``reasoning_tokens: 0`` measurement (None vs 0): a reported 0 is a
+ measurement, an absent object is an unknown, and the two must not collapse
+ into the same value.
+ """
+ details = (usage or {}).get(details_key)
+ if not isinstance(details, dict):
+ return None
+ value = details.get("reasoning_tokens")
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
+ return None
+ return value
+
+
+def set_reasoning_subset_attributes(span, output_tokens, reasoning_tokens):
+ """Emit reasoning-token attributes under the subset invariant.
+
+ ``gen_ai.usage.reasoning.output_tokens`` is a facet of the output total,
+ never an addend. Three states stay distinguishable:
+
+ * provider reported a value (including an explicit 0): the facet is
+ emitted, ``...output_tokens.reported`` is true, and the derived
+ completed figure is available;
+ * provider sent no details object: no facet value, marker false
+ (unknown, not a guessed 0);
+ * no usage at all (stream without include_usage): nothing emitted.
+
+ The completed figure is output minus reasoning and lives under the
+ ``gen_ai.usage.derived.*`` namespace so it can never be mistaken for a
+ provider-reported number.
+ """
+ if reasoning_tokens is None:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,
+ False,
+ )
+ return
+ span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,
+ True,
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,
+ reasoning_tokens,
+ )
+ # OpenLIT legacy alias (pre-OTel naming), kept for backward compat; only
+ # nonzero values so legacy consumers see no behavior change.
+ if reasoning_tokens > 0:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,
+ reasoning_tokens,
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS,
+ max(output_tokens - reasoning_tokens, 0),
+ )
+
+
def truncate_message_content(messages):
"""Apply truncation to text content fields within OTel message structures.
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
index b4768a7b..4556f383 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
@@ -66,7 +66,9 @@ def async_complete(
self._tools = None
self._input_tokens = 0
self._output_tokens = 0
- self._reasoning_tokens = 0
+ # Do not default _reasoning_tokens. Absence means no usage was
+ # received yet; a literal 0 is a measured zero and would be emitted
+ # as reported=true if the stream ends without a usage chunk.
self._cache_read_input_tokens = 0
self._cache_creation_input_tokens = 0
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
index 5c9da4c4..0d9430ed 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
@@ -66,7 +66,9 @@ def complete(
self._tools = None
self._input_tokens = 0
self._output_tokens = 0
- self._reasoning_tokens = 0
+ # Do not default _reasoning_tokens. Absence means no usage was
+ # received yet; a literal 0 is a measured zero and would be emitted
+ # as reported=true if the stream ends without a usage chunk.
self._cache_read_input_tokens = 0
self._cache_creation_input_tokens = 0
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
index f0f045c3..b534b625 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
@@ -21,6 +21,8 @@ from openlit.__helpers import (
general_tokens,
otel_event,
truncate_message_content,
+ extract_reasoning_tokens,
+ set_reasoning_subset_attributes,
)
from openlit._config import OpenlitConfig
from openlit.semcov import SemanticConvention
@@ -350,6 +352,21 @@ def emit_inference_event(
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
+def _reasoning_tokens_from_usage(usage):
+ """Return reasoning tokens (a subset of completion_tokens) from usage.
+
+ None means the provider did not report them, which is not a measured 0.
+ The top-level ``usage.reasoning_tokens`` fallback gets the same numeric
+ check as ``completion_tokens_details``.
+ """
+ reasoning_tokens = extract_reasoning_tokens(usage, "completion_tokens_details")
+ if reasoning_tokens is None:
+ value = usage.get("reasoning_tokens")
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ reasoning_tokens = value
+ return reasoning_tokens
+
+
def process_chunk(scope, chunk):
"""
Process a chunk of response data and update state.
@@ -437,11 +454,7 @@ def process_chunk(scope, chunk):
scope._cache_creation_input_tokens = (
usage.get("input_tokens_details") or {}
).get("cache_creation_tokens", 0) or 0
- completion_details = usage.get("completion_tokens_details", {})
- if "reasoning_tokens" in completion_details:
- scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)
- elif "reasoning_tokens" in usage:
- scope._reasoning_tokens = usage.get("reasoning_tokens", 0)
+ scope._reasoning_tokens = _reasoning_tokens_from_usage(usage)
def common_chat_logic(
@@ -597,14 +610,13 @@ def common_chat_logic(
)
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
- # Span Attributes for Reasoning (if present)
- if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens
- )
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
- input_tokens + output_tokens + scope._reasoning_tokens,
+ # Reasoning tokens (OTel: gen_ai.usage.reasoning.output_tokens is a subset
+ # of gen_ai.usage.output_tokens above, so it is recorded separately and
+ # never added on top of the output total / gen_ai.client.token.usage).
+ # Missing details surface as unknown (marker false), never as a guessed 0.
+ if hasattr(scope, "_reasoning_tokens"):
+ set_reasoning_subset_attributes(
+ scope._span, output_tokens, scope._reasoning_tokens
)
# OTel cached token attributes (set even when 0)
@@ -828,13 +840,7 @@ def process_chat_response(
scope._cache_creation_input_tokens = (usage.get("input_tokens_details") or {}).get(
"cache_creation_tokens", 0
) or 0
- completion_details = usage.get("completion_tokens_details", {})
- if "reasoning_tokens" in completion_details:
- scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)
- elif "reasoning_tokens" in response_dict.get("usage", {}):
- scope._reasoning_tokens = response_dict.get("usage").get("reasoning_tokens", 0)
- else:
- scope._reasoning_tokens = 0
+ scope._reasoning_tokens = _reasoning_tokens_from_usage(usage)
scope._response_id = response_dict.get("id")
scope._response_model = response_dict.get("model")
scope._finish_reason = str(
diff --git a/sdk/python/src/openlit/instrumentation/langchain/utils.py b/sdk/python/src/openlit/instrumentation/langchain/utils.py
index d5a37db9..5dd27c44 100644
--- a/sdk/python/src/openlit/instrumentation/langchain/utils.py
+++ b/sdk/python/src/openlit/instrumentation/langchain/utils.py
@@ -19,6 +19,7 @@ from openlit.__helpers import (
general_tokens,
otel_event,
truncate_message_content,
+ set_reasoning_subset_attributes,
)
from openlit.semcov import SemanticConvention
@@ -492,18 +493,12 @@ def common_chat_logic(
)
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
- # Reasoning tokens (optional)
- if (
- hasattr(scope, "_reasoning_tokens")
- and scope._reasoning_tokens
- and scope._reasoning_tokens > 0
- ):
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens
- )
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
- input_tokens + output_tokens + scope._reasoning_tokens,
+ # Reasoning tokens (optional). OTel: gen_ai.usage.reasoning.output_tokens is
+ # a subset of gen_ai.usage.output_tokens above, so it is recorded separately
+ # and never added on top of the output total / gen_ai.client.token.usage.
+ if hasattr(scope, "_reasoning_tokens"):
+ set_reasoning_subset_attributes(
+ scope._span, output_tokens, scope._reasoning_tokens
)
# OTel cached token attributes (set even when 0)
diff --git a/sdk/python/src/openlit/instrumentation/openai/utils.py b/sdk/python/src/openlit/instrumentation/openai/utils.py
index ff0a7029..bd3e926a 100644
--- a/sdk/python/src/openlit/instrumentation/openai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/openai/utils.py
@@ -31,6 +31,8 @@ from openlit.__helpers import (
common_span_attributes,
otel_event,
truncate_message_content,
+ extract_reasoning_tokens,
+ set_reasoning_subset_attributes,
)
from openlit.semcov import SemanticConvention
from openlit._config import OpenlitConfig
@@ -104,70 +106,6 @@ def extract_reasoning_content(payload):
return reasoning if isinstance(reasoning, str) else ""
-def extract_reasoning_tokens(usage, details_key):
- """Return reasoning tokens (a subset of output tokens) from a usage dict,
- or None when the provider did not report the details object.
-
- Works for both chat completions (``completion_tokens_details``) and the
- responses API (``output_tokens_details``). A missing or non-dict details
- object means "unknown" and stays distinguishable from an explicit
- ``reasoning_tokens: 0`` measurement (None vs 0): a reported 0 is a
- measurement, an absent object is an unknown, and the two must not collapse
- into the same value.
- """
- details = (usage or {}).get(details_key)
- if not isinstance(details, dict):
- return None
- value = details.get("reasoning_tokens")
- if not isinstance(value, (int, float)) or isinstance(value, bool):
- return None
- return value
-
-
-def set_reasoning_subset_attributes(span, output_tokens, reasoning_tokens):
- """Emit reasoning-token attributes under the subset invariant.
-
- ``gen_ai.usage.reasoning.output_tokens`` is a facet of the output total,
- never an addend. Three states stay distinguishable:
-
- * provider reported a value (including an explicit 0): the facet is
- emitted, ``...output_tokens.reported`` is true, and the derived
- completed figure is available;
- * provider sent no details object: no facet value, marker false
- (unknown, not a guessed 0);
- * no usage at all (stream without include_usage): nothing emitted.
-
- The completed figure is output minus reasoning and lives under the
- ``gen_ai.usage.derived.*`` namespace so it can never be mistaken for a
- provider-reported number.
- """
- if reasoning_tokens is None:
- span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,
- False,
- )
- return
- span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,
- True,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,
- reasoning_tokens,
- )
- # OpenLIT legacy alias (pre-OTel naming), kept for backward compat; only
- # nonzero values so legacy consumers see no behavior change.
- if reasoning_tokens > 0:
- span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,
- reasoning_tokens,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS,
- max(output_tokens - reasoning_tokens, 0),
- )
-
-
def format_content(messages):
"""
Format the messages into a string for span events.
diff --git a/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.py b/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.py
new file mode 100644
index 00000000..6e1aaec8
--- /dev/null
+++ b/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.py
@@ -0,0 +1,464 @@
+# pylint: disable=protected-access
+"""Invariant tests for Azure AI Inference reasoning token telemetry (#1537).
+
+Azure AI Inference returns OpenAI-shaped usage, where reasoning tokens are a
+*subset* of the completion tokens: ``prompt_tokens=10`` /
+``completion_tokens=1000`` / ``completion_tokens_details.reasoning_tokens=700``
+means 700 of the 1000 output tokens were spent on reasoning.
+
+The span therefore must record:
+ * ``gen_ai.usage.output_tokens`` = 1000 (already includes the subset)
+ * ``gen_ai.usage.reasoning.output_tokens`` = 700 (the subset, a facet)
+ * ``gen_ai.client.token.usage`` = 1010 (input + output; it used to be
+ overwritten with input + output + reasoning = 1710)
+
+and ``gen_ai.client.token.usage{gen_ai.token.type=output}`` stays 1000. These
+tests lock in the same invariant the OpenAI path follows since #1476/#1543,
+for both the non-streaming and the streaming paths.
+"""
+
+import time
+from unittest.mock import MagicMock
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.azure_ai_inference import (
+ async_azure_ai_inference as async_mod,
+)
+from openlit.instrumentation.azure_ai_inference import azure_ai_inference as sync_mod
+from openlit.instrumentation.azure_ai_inference.utils import process_chat_response
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {
+ "model": "o3-mini",
+ "messages": [{"role": "user", "content": "think step by step"}],
+}
+
+USAGE = {
+ "prompt_tokens": 10,
+ "completion_tokens": 1000,
+ "total_tokens": 1010,
+ "completion_tokens_details": {"reasoning_tokens": 700},
+}
+
+USAGE_WITHOUT_DETAILS = {"prompt_tokens": 10, "completion_tokens": 1000}
+
+
+def _tracer_and_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return (
+ tracer_provider.get_tracer("test-azure-ai-inference-reasoning-tokens"),
+ exporter,
+ )
+
+
+def _metrics_dict():
+ """MagicMock metric instruments mirroring openlit.otel.metrics.build_metrics()."""
+ return {
+ "genai_client_usage_tokens": MagicMock(),
+ "genai_client_operation_duration": MagicMock(),
+ "genai_client_time_to_first_chunk": MagicMock(),
+ "genai_client_time_per_output_chunk": MagicMock(),
+ "genai_server_tbt": MagicMock(),
+ "genai_server_ttft": MagicMock(),
+ "genai_server_request_duration": MagicMock(),
+ "genai_cost": MagicMock(),
+ }
+
+
+def _output_token_usage_records(metrics):
+ """Return [(value, attrs)] of gen_ai.client.token.usage{token_type=output}."""
+ records = []
+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:
+ value, attrs = call.args
+ if (
+ attrs.get(SemanticConvention.GEN_AI_TOKEN_TYPE)
+ == SemanticConvention.GEN_AI_TOKEN_TYPE_OUTPUT
+ ):
+ records.append((value, attrs))
+ return records
+
+
+def _assert_no_double_counting(
+ attrs, metrics, input_tokens=10, output_tokens=1000, reasoning_tokens=700
+):
+ """Shared invariant: reasoning is a subset of output, never added on top."""
+ # Span attributes
+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == input_tokens
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == output_tokens
+ # gen_ai.client.token.usage on the span must stay 1010, not 1710
+ assert (
+ attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]
+ == input_tokens + output_tokens
+ )
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
+ == reasoning_tokens
+ )
+ # OpenLIT legacy alias (pre-OTel naming) must stay populated for backward
+ # compatibility.
+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == reasoning_tokens
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True
+ )
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS]
+ == output_tokens - reasoning_tokens
+ )
+
+ # gen_ai.client.token.usage{token_type=output} must stay 1000, not 1700
+ output_records = _output_token_usage_records(metrics)
+ assert output_records, "expected at least one output token usage record"
+ for value, _attrs in output_records:
+ assert value == output_tokens
+
+ # No recorded value may reach an aggregate that adds reasoning on top.
+ forbidden = (
+ output_tokens + reasoning_tokens,
+ input_tokens + output_tokens + reasoning_tokens,
+ )
+ assert all(value not in forbidden for value in attrs.values())
+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:
+ assert call.args[0] not in forbidden
+
+
+def _assert_reasoning_unknown(attrs, metrics):
+ """Usage without a reasoning count: unknown (marker false), not a guessed 0."""
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000
+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is False
+ )
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs
+ output_records = _output_token_usage_records(metrics)
+ assert output_records, "expected at least one output token usage record"
+ for value, _attrs in output_records:
+ assert value == 1000
+
+
+def _assert_no_reasoning_usage_attrs(attrs):
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs
+
+
+def _chat_response(usage):
+ return {
+ "id": "chatcmpl_o3",
+ "model": "o3-mini",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "The answer is 42."},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": usage,
+ }
+
+
+def _run_chat_response(response, metrics):
+ tracer, exporter = _tracer_and_exporter()
+ with tracer.start_as_current_span("chat o3-mini") as span:
+ process_chat_response(
+ response,
+ request_model="o3-mini",
+ pricing_info={},
+ server_port=443,
+ server_address="models.github.ai",
+ environment="test-env",
+ application_name="test-app",
+ metrics=metrics,
+ start_time=time.time(),
+ span=span,
+ capture_message_content=False,
+ disable_metrics=False,
+ version="test-version",
+ **REQUEST_KWARGS,
+ )
+ return exporter.get_finished_spans()[0].attributes
+
+
+def _stream_chunks(usage):
+ chunks = [
+ {
+ "id": "chatcmpl_o3",
+ "model": "o3-mini",
+ "choices": [
+ {
+ "index": 0,
+ "delta": {"role": "assistant", "content": "The answer is 42."},
+ "finish_reason": None,
+ }
+ ],
+ },
+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
+ ]
+ if usage is not None:
+ # The final chunk carries usage.
+ chunks.append({"choices": [], "usage": usage})
+ return chunks
+
+
+class FakeSyncStream:
+ """Sync context-manager stream yielding Azure AI Inference-shaped chunks."""
+
+ def __init__(self, chunks):
+ self._it = iter(chunks)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self._it)
+
+
+class FakeAsyncStream:
+ """Async context-manager stream yielding Azure AI Inference-shaped chunks."""
+
+ def __init__(self, chunks):
+ self._it = iter(chunks)
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._it)
+ except StopIteration as exc:
+ raise StopAsyncIteration from exc
+
+
+def _factory(tracer, metrics, *, is_async):
+ make = async_mod.async_complete if is_async else sync_mod.complete
+ return make(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=metrics,
+ disable_metrics=False,
+ event_provider=None,
+ )
+
+
+def _finished_stream_span_attributes(stream, exporter):
+ # Stream span lifecycle is out of scope here: end the span if the wrapper
+ # left it open so the exporter sees the attributes.
+ if stream._span.end_time is None:
+ stream._span.end()
+ return exporter.get_finished_spans()[0].attributes
+
+
+def _run_sync_stream(usage, metrics):
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = _factory(tracer, metrics, is_async=False)
+ stream = wrapper(
+ lambda *a, **k: FakeSyncStream(_stream_chunks(usage)),
+ None,
+ (),
+ {**REQUEST_KWARGS, "stream": True},
+ )
+ with stream as s:
+ for _ in s:
+ pass
+ return _finished_stream_span_attributes(stream, exporter)
+
+
+async def _run_async_stream(usage, metrics):
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = _factory(tracer, metrics, is_async=True)
+
+ async def _create(*_args, **_kwargs):
+ return FakeAsyncStream(_stream_chunks(usage))
+
+ stream = await wrapper(_create, None, (), {**REQUEST_KWARGS, "stream": True})
+ async with stream as s:
+ async for _ in s:
+ pass
+ return _finished_stream_span_attributes(stream, exporter)
+
+
+def test_chat_reasoning_tokens_are_subset_of_output():
+ """prompt 10 / completion 1000 / reasoning 700: token usage is 1010, not 1710."""
+ metrics = _metrics_dict()
+ attrs = _run_chat_response(_chat_response(USAGE), metrics)
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+def test_chat_sdk_response_object_keeps_subset():
+ """Real azure-ai-inference ChatCompletions: usage is a Mapping, not a dict."""
+ models = pytest.importorskip("azure.ai.inference.models")
+ metrics = _metrics_dict()
+ response = models.ChatCompletions({"created": 1, **_chat_response(USAGE)})
+ attrs = _run_chat_response(response, metrics)
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+def test_streaming_chat_reasoning_tokens_are_subset_of_output():
+ """Streaming chat with usage on the final chunk: reasoning is a subset."""
+ metrics = _metrics_dict()
+ attrs = _run_sync_stream(USAGE, metrics)
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+async def test_async_streaming_chat_reasoning_tokens_are_subset_of_output():
+ """Async streaming chat with usage on the final chunk: reasoning is a subset."""
+ metrics = _metrics_dict()
+ attrs = await _run_async_stream(USAGE, metrics)
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+def test_no_completion_tokens_details_marks_reasoning_unknown():
+ """No completion_tokens_details: facet value absent, reported marker false."""
+ metrics = _metrics_dict()
+ attrs = _run_chat_response(_chat_response(USAGE_WITHOUT_DETAILS), metrics)
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+def test_null_completion_tokens_details_does_not_crash():
+ """completion_tokens_details: null is unknown, not a crash or a guessed 0."""
+ metrics = _metrics_dict()
+ attrs = _run_chat_response(
+ _chat_response({**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}),
+ metrics,
+ )
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+def test_streaming_no_completion_tokens_details_marks_reasoning_unknown():
+ """Stream usage chunk without completion_tokens_details: reasoning unknown."""
+ metrics = _metrics_dict()
+ attrs = _run_sync_stream(USAGE_WITHOUT_DETAILS, metrics)
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+def test_streaming_null_completion_tokens_details_does_not_crash():
+ """Stream usage chunk with completion_tokens_details: null.
+
+ process_chunk runs inside the caller's iteration with no exception guard,
+ so a failure here would break the application's stream loop.
+ """
+ metrics = _metrics_dict()
+ attrs = _run_sync_stream(
+ {**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}, metrics
+ )
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+async def test_async_streaming_null_completion_tokens_details_does_not_crash():
+ """Async stream usage chunk with completion_tokens_details: null."""
+ metrics = _metrics_dict()
+ attrs = await _run_async_stream(
+ {**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}, metrics
+ )
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+def test_top_level_reasoning_tokens_fallback_is_subset_of_output():
+ """usage.reasoning_tokens fallback: still a subset, never added on top."""
+ metrics = _metrics_dict()
+ attrs = _run_chat_response(
+ _chat_response({**USAGE_WITHOUT_DETAILS, "reasoning_tokens": 700}), metrics
+ )
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+def test_streaming_top_level_reasoning_tokens_fallback_is_subset_of_output():
+ """Stream usage chunk with usage.reasoning_tokens: still a subset."""
+ metrics = _metrics_dict()
+ attrs = _run_sync_stream(
+ {**USAGE_WITHOUT_DETAILS, "reasoning_tokens": 700}, metrics
+ )
+
+ _assert_no_double_counting(attrs, metrics)
+
+
+def test_streaming_non_numeric_reasoning_tokens_does_not_crash():
+ """Non-numeric usage.reasoning_tokens is unknown, not a crash on stream exit."""
+ metrics = _metrics_dict()
+ attrs = _run_sync_stream(
+ {**USAGE_WITHOUT_DETAILS, "reasoning_tokens": "700"}, metrics
+ )
+
+ _assert_reasoning_unknown(attrs, metrics)
+
+
+def test_zero_reasoning_tokens_is_a_measurement():
+ """reasoning_tokens explicitly 0: a measurement (facet 0), not unknown."""
+ metrics = _metrics_dict()
+ attrs = _run_chat_response(
+ _chat_response(
+ {
+ "prompt_tokens": 10,
+ "completion_tokens": 1000,
+ "completion_tokens_details": {"reasoning_tokens": 0},
+ }
+ ),
+ metrics,
+ )
+
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000
+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010
+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 0
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True
+ )
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS] == 1000
+ )
+ # The legacy alias stays nonzero-only, as on the OpenAI path.
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs
+
+
+def test_streaming_chat_without_usage_emits_no_reasoning_attrs():
+ """Stream that never sends usage: no reasoning attributes at all."""
+ attrs = _run_sync_stream(None, _metrics_dict())
+
+ _assert_no_reasoning_usage_attrs(attrs)
+
+
+async def test_async_streaming_chat_without_usage_emits_no_reasoning_attrs():
+ """Async stream that never sends usage: no reasoning attributes at all."""
+ attrs = await _run_async_stream(None, _metrics_dict())
+
+ _assert_no_reasoning_usage_attrs(attrs)
diff --git a/sdk/python/tests/test_langchain_reasoning_tokens.py b/sdk/python/tests/test_langchain_reasoning_tokens.py
new file mode 100644
index 00000000..b9afcfba
--- /dev/null
+++ b/sdk/python/tests/test_langchain_reasoning_tokens.py
@@ -0,0 +1,124 @@
+"""Invariant tests for LangChain reasoning token telemetry (#1537).
+
+LangChain's normalized usage reports reasoning as a breakdown of
+``output_tokens``, so reasoning tokens are a subset of the output total and
+must never be added on top of ``gen_ai.client.token.usage``. The callback
+handler does not populate ``scope._reasoning_tokens`` today; these tests pin
+``common_chat_logic`` to the subset invariant for when it does, and pin the
+current callback scope to emitting no reasoning attributes.
+"""
+
+import time
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.langchain.utils import common_chat_logic
+from openlit.semcov import SemanticConvention
+
+
+def _tracer_and_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return tracer_provider.get_tracer("test-langchain-reasoning-tokens"), exporter
+
+
+def _metrics_dict():
+ """MagicMock metric instruments mirroring openlit.otel.metrics.build_metrics()."""
+ return {
+ "genai_client_usage_tokens": MagicMock(),
+ "genai_client_operation_duration": MagicMock(),
+ "genai_client_time_to_first_chunk": MagicMock(),
+ "genai_client_time_per_output_chunk": MagicMock(),
+ "genai_server_tbt": MagicMock(),
+ "genai_server_ttft": MagicMock(),
+ "genai_server_request_duration": MagicMock(),
+ "genai_cost": MagicMock(),
+ }
+
+
+def _callback_scope(span, **extra):
+ """Mirrors the scope OpenLITCallbackHandler builds before common_chat_logic."""
+ now = time.time()
+ return SimpleNamespace(
+ _span=span,
+ _kwargs={},
+ _model_parameters={},
+ _start_time=now,
+ _end_time=now,
+ _server_address="api.openai.com",
+ _server_port=443,
+ _response_model="o3-mini",
+ _response_id="chatcmpl_o3",
+ _llmresponse="The answer is 42.",
+ _finish_reason="stop",
+ _tools=None,
+ _input_tokens=10,
+ _output_tokens=1000,
+ _cache_read_input_tokens=0,
+ _cache_creation_input_tokens=0,
+ _timestamps=[],
+ _tbt=0,
+ _ttft=0,
+ _request_model="o3-mini",
+ _input_messages_raw=None,
+ _prompts=["think step by step"],
+ _system_instructions=None,
+ _tool_definitions=None,
+ _provider="openai",
+ **extra,
+ )
+
+
+def _run_common_chat_logic(metrics, **extra):
+ tracer, exporter = _tracer_and_exporter()
+ with tracer.start_as_current_span("chat o3-mini") as span:
+ common_chat_logic(
+ _callback_scope(span, **extra),
+ pricing_info={},
+ environment="test-env",
+ application_name="test-app",
+ metrics=metrics,
+ capture_message_content=False,
+ disable_metrics=False,
+ version="test-version",
+ is_stream=False,
+ )
+ return exporter.get_finished_spans()[0].attributes
+
+
+def test_reasoning_tokens_are_subset_of_output():
+ """prompt 10 / output 1000 / reasoning 700: token usage is 1010, not 1710."""
+ metrics = _metrics_dict()
+ attrs = _run_common_chat_logic(metrics, _reasoning_tokens=700)
+
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000
+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010
+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 700
+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == 700
+ assert (
+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True
+ )
+ assert attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS] == 300
+ assert all(value not in (1700, 1710) for value in attrs.values())
+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:
+ assert call.args[0] not in (1700, 1710)
+
+
+def test_callback_scope_without_reasoning_tokens_emits_no_reasoning_attrs():
+ """The callback handler sets no _reasoning_tokens: nothing is emitted."""
+ attrs = _run_common_chat_logic(_metrics_dict())
+
+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs