Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(azure_ai_inference): keep reasoning tokens out of gen_ai.client.token.usage (#1595)
Dmitry Voropaev42e11e4a115a
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 2b1ac2ee..e4ee415e 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -238,6 +238,70 @@ def set_span_reasoning_content(span, scope, capture_message_content=True):)+def extract_reasoning_tokens(usage, details_key):+ """Return reasoning tokens (a subset of output tokens) from a usage dict,+ or None when the provider did not report the details object.++ Works for both chat completions (``completion_tokens_details``) and the+ responses API (``output_tokens_details``). A missing or non-dict details+ object means "unknown" and stays distinguishable from an explicit+ ``reasoning_tokens: 0`` measurement (None vs 0): a reported 0 is a+ measurement, an absent object is an unknown, and the two must not collapse+ into the same value.+ """+ details = (usage or {}).get(details_key)+ if not isinstance(details, dict):+ return None+ value = details.get("reasoning_tokens")+ if not isinstance(value, (int, float)) or isinstance(value, bool):+ return None+ return value+++def set_reasoning_subset_attributes(span, output_tokens, reasoning_tokens):+ """Emit reasoning-token attributes under the subset invariant.++ ``gen_ai.usage.reasoning.output_tokens`` is a facet of the output total,+ never an addend. Three states stay distinguishable:++ * provider reported a value (including an explicit 0): the facet is+ emitted, ``...output_tokens.reported`` is true, and the derived+ completed figure is available;+ * provider sent no details object: no facet value, marker false+ (unknown, not a guessed 0);+ * no usage at all (stream without include_usage): nothing emitted.++ The completed figure is output minus reasoning and lives under the+ ``gen_ai.usage.derived.*`` namespace so it can never be mistaken for a+ provider-reported number.+ """+ if reasoning_tokens is None:+ span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,+ False,+ )+ return+ span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,+ True,+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,+ reasoning_tokens,+ )+ # OpenLIT legacy alias (pre-OTel naming), kept for backward compat; only+ # nonzero values so legacy consumers see no behavior change.+ if reasoning_tokens > 0:+ span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,+ reasoning_tokens,+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS,+ max(output_tokens - reasoning_tokens, 0),+ )++def truncate_message_content(messages):"""Apply truncation to text content fields within OTel message structures.diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.pyindex b4768a7b..4556f383 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py@@ -66,7 +66,9 @@ def async_complete(self._tools = Noneself._input_tokens = 0self._output_tokens = 0- self._reasoning_tokens = 0+ # Do not default _reasoning_tokens. Absence means no usage was+ # received yet; a literal 0 is a measured zero and would be emitted+ # as reported=true if the stream ends without a usage chunk.self._cache_read_input_tokens = 0self._cache_creation_input_tokens = 0diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.pyindex 5c9da4c4..0d9430ed 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py@@ -66,7 +66,9 @@ def complete(self._tools = Noneself._input_tokens = 0self._output_tokens = 0- self._reasoning_tokens = 0+ # Do not default _reasoning_tokens. Absence means no usage was+ # received yet; a literal 0 is a measured zero and would be emitted+ # as reported=true if the stream ends without a usage chunk.self._cache_read_input_tokens = 0self._cache_creation_input_tokens = 0diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.pyindex f0f045c3..b534b625 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py@@ -21,6 +21,8 @@ from openlit.__helpers import (general_tokens,otel_event,truncate_message_content,+ extract_reasoning_tokens,+ set_reasoning_subset_attributes,)from openlit._config import OpenlitConfigfrom openlit.semcov import SemanticConvention@@ -350,6 +352,21 @@ def emit_inference_event(logger.warning("Failed to emit inference event: %s", e, exc_info=True)+def _reasoning_tokens_from_usage(usage):+ """Return reasoning tokens (a subset of completion_tokens) from usage.++ None means the provider did not report them, which is not a measured 0.+ The top-level ``usage.reasoning_tokens`` fallback gets the same numeric+ check as ``completion_tokens_details``.+ """+ reasoning_tokens = extract_reasoning_tokens(usage, "completion_tokens_details")+ if reasoning_tokens is None:+ value = usage.get("reasoning_tokens")+ if isinstance(value, (int, float)) and not isinstance(value, bool):+ reasoning_tokens = value+ return reasoning_tokens++def process_chunk(scope, chunk):"""Process a chunk of response data and update state.@@ -437,11 +454,7 @@ def process_chunk(scope, chunk):scope._cache_creation_input_tokens = (usage.get("input_tokens_details") or {}).get("cache_creation_tokens", 0) or 0- completion_details = usage.get("completion_tokens_details", {})- if "reasoning_tokens" in completion_details:- scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)- elif "reasoning_tokens" in usage:- scope._reasoning_tokens = usage.get("reasoning_tokens", 0)+ scope._reasoning_tokens = _reasoning_tokens_from_usage(usage)def common_chat_logic(@@ -597,14 +610,13 @@ def common_chat_logic()scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)- # Span Attributes for Reasoning (if present)- if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:- scope._span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens- )- scope._span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,- input_tokens + output_tokens + scope._reasoning_tokens,+ # Reasoning tokens (OTel: gen_ai.usage.reasoning.output_tokens is a subset+ # of gen_ai.usage.output_tokens above, so it is recorded separately and+ # never added on top of the output total / gen_ai.client.token.usage).+ # Missing details surface as unknown (marker false), never as a guessed 0.+ if hasattr(scope, "_reasoning_tokens"):+ set_reasoning_subset_attributes(+ scope._span, output_tokens, scope._reasoning_tokens)# OTel cached token attributes (set even when 0)@@ -828,13 +840,7 @@ def process_chat_response(scope._cache_creation_input_tokens = (usage.get("input_tokens_details") or {}).get("cache_creation_tokens", 0) or 0- completion_details = usage.get("completion_tokens_details", {})- if "reasoning_tokens" in completion_details:- scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)- elif "reasoning_tokens" in response_dict.get("usage", {}):- scope._reasoning_tokens = response_dict.get("usage").get("reasoning_tokens", 0)- else:- scope._reasoning_tokens = 0+ scope._reasoning_tokens = _reasoning_tokens_from_usage(usage)scope._response_id = response_dict.get("id")scope._response_model = response_dict.get("model")scope._finish_reason = str(diff --git a/sdk/python/src/openlit/instrumentation/langchain/utils.py b/sdk/python/src/openlit/instrumentation/langchain/utils.pyindex d5a37db9..5dd27c44 100644--- a/sdk/python/src/openlit/instrumentation/langchain/utils.py+++ b/sdk/python/src/openlit/instrumentation/langchain/utils.py@@ -19,6 +19,7 @@ from openlit.__helpers import (general_tokens,otel_event,truncate_message_content,+ set_reasoning_subset_attributes,)from openlit.semcov import SemanticConvention@@ -492,18 +493,12 @@ def common_chat_logic()scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)- # Reasoning tokens (optional)- if (- hasattr(scope, "_reasoning_tokens")- and scope._reasoning_tokens- and scope._reasoning_tokens > 0- ):- scope._span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens- )- scope._span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,- input_tokens + output_tokens + scope._reasoning_tokens,+ # Reasoning tokens (optional). OTel: gen_ai.usage.reasoning.output_tokens is+ # a subset of gen_ai.usage.output_tokens above, so it is recorded separately+ # and never added on top of the output total / gen_ai.client.token.usage.+ if hasattr(scope, "_reasoning_tokens"):+ set_reasoning_subset_attributes(+ scope._span, output_tokens, scope._reasoning_tokens)# OTel cached token attributes (set even when 0)diff --git a/sdk/python/src/openlit/instrumentation/openai/utils.py b/sdk/python/src/openlit/instrumentation/openai/utils.pyindex ff0a7029..bd3e926a 100644--- a/sdk/python/src/openlit/instrumentation/openai/utils.py+++ b/sdk/python/src/openlit/instrumentation/openai/utils.py@@ -31,6 +31,8 @@ from openlit.__helpers import (common_span_attributes,otel_event,truncate_message_content,+ extract_reasoning_tokens,+ set_reasoning_subset_attributes,)from openlit.semcov import SemanticConventionfrom openlit._config import OpenlitConfig@@ -104,70 +106,6 @@ def extract_reasoning_content(payload):return reasoning if isinstance(reasoning, str) else ""-def extract_reasoning_tokens(usage, details_key):- """Return reasoning tokens (a subset of output tokens) from a usage dict,- or None when the provider did not report the details object.-- Works for both chat completions (``completion_tokens_details``) and the- responses API (``output_tokens_details``). A missing or non-dict details- object means "unknown" and stays distinguishable from an explicit- ``reasoning_tokens: 0`` measurement (None vs 0): a reported 0 is a- measurement, an absent object is an unknown, and the two must not collapse- into the same value.- """- details = (usage or {}).get(details_key)- if not isinstance(details, dict):- return None- value = details.get("reasoning_tokens")- if not isinstance(value, (int, float)) or isinstance(value, bool):- return None- return value---def set_reasoning_subset_attributes(span, output_tokens, reasoning_tokens):- """Emit reasoning-token attributes under the subset invariant.-- ``gen_ai.usage.reasoning.output_tokens`` is a facet of the output total,- never an addend. Three states stay distinguishable:-- * provider reported a value (including an explicit 0): the facet is- emitted, ``...output_tokens.reported`` is true, and the derived- completed figure is available;- * provider sent no details object: no facet value, marker false- (unknown, not a guessed 0);- * no usage at all (stream without include_usage): nothing emitted.-- The completed figure is output minus reasoning and lives under the- ``gen_ai.usage.derived.*`` namespace so it can never be mistaken for a- provider-reported number.- """- if reasoning_tokens is None:- span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,- False,- )- return- span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED,- True,- )- span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,- reasoning_tokens,- )- # OpenLIT legacy alias (pre-OTel naming), kept for backward compat; only- # nonzero values so legacy consumers see no behavior change.- if reasoning_tokens > 0:- span.set_attribute(- SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,- reasoning_tokens,- )- span.set_attribute(- SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS,- max(output_tokens - reasoning_tokens, 0),- )--def format_content(messages):"""Format the messages into a string for span events.diff --git a/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.py b/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.pynew file mode 100644index 00000000..6e1aaec8--- /dev/null+++ b/sdk/python/tests/test_azure_ai_inference_reasoning_tokens.py@@ -0,0 +1,464 @@+# pylint: disable=protected-access+"""Invariant tests for Azure AI Inference reasoning token telemetry (#1537).++Azure AI Inference returns OpenAI-shaped usage, where reasoning tokens are a+*subset* of the completion tokens: ``prompt_tokens=10`` /+``completion_tokens=1000`` / ``completion_tokens_details.reasoning_tokens=700``+means 700 of the 1000 output tokens were spent on reasoning.++The span therefore must record:+ * ``gen_ai.usage.output_tokens`` = 1000 (already includes the subset)+ * ``gen_ai.usage.reasoning.output_tokens`` = 700 (the subset, a facet)+ * ``gen_ai.client.token.usage`` = 1010 (input + output; it used to be+ overwritten with input + output + reasoning = 1710)++and ``gen_ai.client.token.usage{gen_ai.token.type=output}`` stays 1000. These+tests lock in the same invariant the OpenAI path follows since #1476/#1543,+for both the non-streaming and the streaming paths.+"""++import time+from unittest.mock import MagicMock++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.azure_ai_inference import (+ async_azure_ai_inference as async_mod,+)+from openlit.instrumentation.azure_ai_inference import azure_ai_inference as sync_mod+from openlit.instrumentation.azure_ai_inference.utils import process_chat_response+from openlit.semcov import SemanticConvention++REQUEST_KWARGS = {+ "model": "o3-mini",+ "messages": [{"role": "user", "content": "think step by step"}],+}++USAGE = {+ "prompt_tokens": 10,+ "completion_tokens": 1000,+ "total_tokens": 1010,+ "completion_tokens_details": {"reasoning_tokens": 700},+}++USAGE_WITHOUT_DETAILS = {"prompt_tokens": 10, "completion_tokens": 1000}+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ return (+ tracer_provider.get_tracer("test-azure-ai-inference-reasoning-tokens"),+ exporter,+ )+++def _metrics_dict():+ """MagicMock metric instruments mirroring openlit.otel.metrics.build_metrics()."""+ return {+ "genai_client_usage_tokens": MagicMock(),+ "genai_client_operation_duration": MagicMock(),+ "genai_client_time_to_first_chunk": MagicMock(),+ "genai_client_time_per_output_chunk": MagicMock(),+ "genai_server_tbt": MagicMock(),+ "genai_server_ttft": MagicMock(),+ "genai_server_request_duration": MagicMock(),+ "genai_cost": MagicMock(),+ }+++def _output_token_usage_records(metrics):+ """Return [(value, attrs)] of gen_ai.client.token.usage{token_type=output}."""+ records = []+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:+ value, attrs = call.args+ if (+ attrs.get(SemanticConvention.GEN_AI_TOKEN_TYPE)+ == SemanticConvention.GEN_AI_TOKEN_TYPE_OUTPUT+ ):+ records.append((value, attrs))+ return records+++def _assert_no_double_counting(+ attrs, metrics, input_tokens=10, output_tokens=1000, reasoning_tokens=700+):+ """Shared invariant: reasoning is a subset of output, never added on top."""+ # Span attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == input_tokens+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == output_tokens+ # gen_ai.client.token.usage on the span must stay 1010, not 1710+ assert (+ attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]+ == input_tokens + output_tokens+ )+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]+ == reasoning_tokens+ )+ # OpenLIT legacy alias (pre-OTel naming) must stay populated for backward+ # compatibility.+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == reasoning_tokens+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True+ )+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS]+ == output_tokens - reasoning_tokens+ )++ # gen_ai.client.token.usage{token_type=output} must stay 1000, not 1700+ output_records = _output_token_usage_records(metrics)+ assert output_records, "expected at least one output token usage record"+ for value, _attrs in output_records:+ assert value == output_tokens++ # No recorded value may reach an aggregate that adds reasoning on top.+ forbidden = (+ output_tokens + reasoning_tokens,+ input_tokens + output_tokens + reasoning_tokens,+ )+ assert all(value not in forbidden for value in attrs.values())+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:+ assert call.args[0] not in forbidden+++def _assert_reasoning_unknown(attrs, metrics):+ """Usage without a reasoning count: unknown (marker false), not a guessed 0."""+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is False+ )+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs+ output_records = _output_token_usage_records(metrics)+ assert output_records, "expected at least one output token usage record"+ for value, _attrs in output_records:+ assert value == 1000+++def _assert_no_reasoning_usage_attrs(attrs):+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED not in attrs+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs+++def _chat_response(usage):+ return {+ "id": "chatcmpl_o3",+ "model": "o3-mini",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": "stop",+ }+ ],+ "usage": usage,+ }+++def _run_chat_response(response, metrics):+ tracer, exporter = _tracer_and_exporter()+ with tracer.start_as_current_span("chat o3-mini") as span:+ process_chat_response(+ response,+ request_model="o3-mini",+ pricing_info={},+ server_port=443,+ server_address="models.github.ai",+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ **REQUEST_KWARGS,+ )+ return exporter.get_finished_spans()[0].attributes+++def _stream_chunks(usage):+ chunks = [+ {+ "id": "chatcmpl_o3",+ "model": "o3-mini",+ "choices": [+ {+ "index": 0,+ "delta": {"role": "assistant", "content": "The answer is 42."},+ "finish_reason": None,+ }+ ],+ },+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},+ ]+ if usage is not None:+ # The final chunk carries usage.+ chunks.append({"choices": [], "usage": usage})+ return chunks+++class FakeSyncStream:+ """Sync context-manager stream yielding Azure AI Inference-shaped chunks."""++ def __init__(self, chunks):+ self._it = iter(chunks)++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def __iter__(self):+ return self++ def __next__(self):+ return next(self._it)+++class FakeAsyncStream:+ """Async context-manager stream yielding Azure AI Inference-shaped chunks."""++ def __init__(self, chunks):+ self._it = iter(chunks)++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self._it)+ except StopIteration as exc:+ raise StopAsyncIteration from exc+++def _factory(tracer, metrics, *, is_async):+ make = async_mod.async_complete if is_async else sync_mod.complete+ return make(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=metrics,+ disable_metrics=False,+ event_provider=None,+ )+++def _finished_stream_span_attributes(stream, exporter):+ # Stream span lifecycle is out of scope here: end the span if the wrapper+ # left it open so the exporter sees the attributes.+ if stream._span.end_time is None:+ stream._span.end()+ return exporter.get_finished_spans()[0].attributes+++def _run_sync_stream(usage, metrics):+ tracer, exporter = _tracer_and_exporter()+ wrapper = _factory(tracer, metrics, is_async=False)+ stream = wrapper(+ lambda *a, **k: FakeSyncStream(_stream_chunks(usage)),+ None,+ (),+ {**REQUEST_KWARGS, "stream": True},+ )+ with stream as s:+ for _ in s:+ pass+ return _finished_stream_span_attributes(stream, exporter)+++async def _run_async_stream(usage, metrics):+ tracer, exporter = _tracer_and_exporter()+ wrapper = _factory(tracer, metrics, is_async=True)++ async def _create(*_args, **_kwargs):+ return FakeAsyncStream(_stream_chunks(usage))++ stream = await wrapper(_create, None, (), {**REQUEST_KWARGS, "stream": True})+ async with stream as s:+ async for _ in s:+ pass+ return _finished_stream_span_attributes(stream, exporter)+++def test_chat_reasoning_tokens_are_subset_of_output():+ """prompt 10 / completion 1000 / reasoning 700: token usage is 1010, not 1710."""+ metrics = _metrics_dict()+ attrs = _run_chat_response(_chat_response(USAGE), metrics)++ _assert_no_double_counting(attrs, metrics)+++def test_chat_sdk_response_object_keeps_subset():+ """Real azure-ai-inference ChatCompletions: usage is a Mapping, not a dict."""+ models = pytest.importorskip("azure.ai.inference.models")+ metrics = _metrics_dict()+ response = models.ChatCompletions({"created": 1, **_chat_response(USAGE)})+ attrs = _run_chat_response(response, metrics)++ _assert_no_double_counting(attrs, metrics)+++def test_streaming_chat_reasoning_tokens_are_subset_of_output():+ """Streaming chat with usage on the final chunk: reasoning is a subset."""+ metrics = _metrics_dict()+ attrs = _run_sync_stream(USAGE, metrics)++ _assert_no_double_counting(attrs, metrics)+++async def test_async_streaming_chat_reasoning_tokens_are_subset_of_output():+ """Async streaming chat with usage on the final chunk: reasoning is a subset."""+ metrics = _metrics_dict()+ attrs = await _run_async_stream(USAGE, metrics)++ _assert_no_double_counting(attrs, metrics)+++def test_no_completion_tokens_details_marks_reasoning_unknown():+ """No completion_tokens_details: facet value absent, reported marker false."""+ metrics = _metrics_dict()+ attrs = _run_chat_response(_chat_response(USAGE_WITHOUT_DETAILS), metrics)++ _assert_reasoning_unknown(attrs, metrics)+++def test_null_completion_tokens_details_does_not_crash():+ """completion_tokens_details: null is unknown, not a crash or a guessed 0."""+ metrics = _metrics_dict()+ attrs = _run_chat_response(+ _chat_response({**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}),+ metrics,+ )++ _assert_reasoning_unknown(attrs, metrics)+++def test_streaming_no_completion_tokens_details_marks_reasoning_unknown():+ """Stream usage chunk without completion_tokens_details: reasoning unknown."""+ metrics = _metrics_dict()+ attrs = _run_sync_stream(USAGE_WITHOUT_DETAILS, metrics)++ _assert_reasoning_unknown(attrs, metrics)+++def test_streaming_null_completion_tokens_details_does_not_crash():+ """Stream usage chunk with completion_tokens_details: null.++ process_chunk runs inside the caller's iteration with no exception guard,+ so a failure here would break the application's stream loop.+ """+ metrics = _metrics_dict()+ attrs = _run_sync_stream(+ {**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}, metrics+ )++ _assert_reasoning_unknown(attrs, metrics)+++async def test_async_streaming_null_completion_tokens_details_does_not_crash():+ """Async stream usage chunk with completion_tokens_details: null."""+ metrics = _metrics_dict()+ attrs = await _run_async_stream(+ {**USAGE_WITHOUT_DETAILS, "completion_tokens_details": None}, metrics+ )++ _assert_reasoning_unknown(attrs, metrics)+++def test_top_level_reasoning_tokens_fallback_is_subset_of_output():+ """usage.reasoning_tokens fallback: still a subset, never added on top."""+ metrics = _metrics_dict()+ attrs = _run_chat_response(+ _chat_response({**USAGE_WITHOUT_DETAILS, "reasoning_tokens": 700}), metrics+ )++ _assert_no_double_counting(attrs, metrics)+++def test_streaming_top_level_reasoning_tokens_fallback_is_subset_of_output():+ """Stream usage chunk with usage.reasoning_tokens: still a subset."""+ metrics = _metrics_dict()+ attrs = _run_sync_stream(+ {**USAGE_WITHOUT_DETAILS, "reasoning_tokens": 700}, metrics+ )++ _assert_no_double_counting(attrs, metrics)+++def test_streaming_non_numeric_reasoning_tokens_does_not_crash():+ """Non-numeric usage.reasoning_tokens is unknown, not a crash on stream exit."""+ metrics = _metrics_dict()+ attrs = _run_sync_stream(+ {**USAGE_WITHOUT_DETAILS, "reasoning_tokens": "700"}, metrics+ )++ _assert_reasoning_unknown(attrs, metrics)+++def test_zero_reasoning_tokens_is_a_measurement():+ """reasoning_tokens explicitly 0: a measurement (facet 0), not unknown."""+ metrics = _metrics_dict()+ attrs = _run_chat_response(+ _chat_response(+ {+ "prompt_tokens": 10,+ "completion_tokens": 1000,+ "completion_tokens_details": {"reasoning_tokens": 0},+ }+ ),+ metrics,+ )++ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 0+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True+ )+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS] == 1000+ )+ # The legacy alias stays nonzero-only, as on the OpenAI path.+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs+++def test_streaming_chat_without_usage_emits_no_reasoning_attrs():+ """Stream that never sends usage: no reasoning attributes at all."""+ attrs = _run_sync_stream(None, _metrics_dict())++ _assert_no_reasoning_usage_attrs(attrs)+++async def test_async_streaming_chat_without_usage_emits_no_reasoning_attrs():+ """Async stream that never sends usage: no reasoning attributes at all."""+ attrs = await _run_async_stream(None, _metrics_dict())++ _assert_no_reasoning_usage_attrs(attrs)diff --git a/sdk/python/tests/test_langchain_reasoning_tokens.py b/sdk/python/tests/test_langchain_reasoning_tokens.pynew file mode 100644index 00000000..b9afcfba--- /dev/null+++ b/sdk/python/tests/test_langchain_reasoning_tokens.py@@ -0,0 +1,124 @@+"""Invariant tests for LangChain reasoning token telemetry (#1537).++LangChain's normalized usage reports reasoning as a breakdown of+``output_tokens``, so reasoning tokens are a subset of the output total and+must never be added on top of ``gen_ai.client.token.usage``. The callback+handler does not populate ``scope._reasoning_tokens`` today; these tests pin+``common_chat_logic`` to the subset invariant for when it does, and pin the+current callback scope to emitting no reasoning attributes.+"""++import time+from types import SimpleNamespace+from unittest.mock import MagicMock++from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.langchain.utils import common_chat_logic+from openlit.semcov import SemanticConvention+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ return tracer_provider.get_tracer("test-langchain-reasoning-tokens"), exporter+++def _metrics_dict():+ """MagicMock metric instruments mirroring openlit.otel.metrics.build_metrics()."""+ return {+ "genai_client_usage_tokens": MagicMock(),+ "genai_client_operation_duration": MagicMock(),+ "genai_client_time_to_first_chunk": MagicMock(),+ "genai_client_time_per_output_chunk": MagicMock(),+ "genai_server_tbt": MagicMock(),+ "genai_server_ttft": MagicMock(),+ "genai_server_request_duration": MagicMock(),+ "genai_cost": MagicMock(),+ }+++def _callback_scope(span, **extra):+ """Mirrors the scope OpenLITCallbackHandler builds before common_chat_logic."""+ now = time.time()+ return SimpleNamespace(+ _span=span,+ _kwargs={},+ _model_parameters={},+ _start_time=now,+ _end_time=now,+ _server_address="api.openai.com",+ _server_port=443,+ _response_model="o3-mini",+ _response_id="chatcmpl_o3",+ _llmresponse="The answer is 42.",+ _finish_reason="stop",+ _tools=None,+ _input_tokens=10,+ _output_tokens=1000,+ _cache_read_input_tokens=0,+ _cache_creation_input_tokens=0,+ _timestamps=[],+ _tbt=0,+ _ttft=0,+ _request_model="o3-mini",+ _input_messages_raw=None,+ _prompts=["think step by step"],+ _system_instructions=None,+ _tool_definitions=None,+ _provider="openai",+ **extra,+ )+++def _run_common_chat_logic(metrics, **extra):+ tracer, exporter = _tracer_and_exporter()+ with tracer.start_as_current_span("chat o3-mini") as span:+ common_chat_logic(+ _callback_scope(span, **extra),+ pricing_info={},+ environment="test-env",+ application_name="test-app",+ metrics=metrics,+ capture_message_content=False,+ disable_metrics=False,+ version="test-version",+ is_stream=False,+ )+ return exporter.get_finished_spans()[0].attributes+++def test_reasoning_tokens_are_subset_of_output():+ """prompt 10 / output 1000 / reasoning 700: token usage is 1010, not 1710."""+ metrics = _metrics_dict()+ attrs = _run_common_chat_logic(metrics, _reasoning_tokens=700)++ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 1000+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 700+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == 700+ assert (+ attrs[SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED] is True+ )+ assert attrs[SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS] == 300+ assert all(value not in (1700, 1710) for value in attrs.values())+ for call in metrics["genai_client_usage_tokens"].record.call_args_list:+ assert call.args[0] not in (1700, 1710)+++def test_callback_scope_without_reasoning_tokens_emits_no_reasoning_attrs():+ """The callback handler sets no _reasoning_tokens: nothing is emitted."""+ attrs = _run_common_chat_logic(_metrics_dict())++ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 1010+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_REPORTED not in attrs+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs