Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(vertexai): track reasoning tokens from thoughts_token_count (#1544)
Goutham Annem9cbcc324e8bc
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.pyindex 370d0c6b..e9f60e89 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py@@ -59,6 +59,7 @@ def async_send_message(self._llmresponse = ""self._input_tokens = 0self._output_tokens = 0+ self._reasoning_tokens = 0self._cache_read_input_tokens = 0self._cache_creation_input_tokens = 0self._kwargs = kwargsdiff --git a/sdk/python/src/openlit/instrumentation/vertexai/utils.py b/sdk/python/src/openlit/instrumentation/vertexai/utils.pyindex 1e6b0cec..9077eb40 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/utils.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/utils.py@@ -229,6 +229,10 @@ def emit_inference_event(attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = valueelif key == "output_tokens":attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = value+ elif key == "reasoning_tokens":+ # Vertex/Gemini: thoughts_token_count is separate from+ # candidates_token_count, not a subset of output tokens.+ attributes[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] = valueelif key == "cache_read_input_tokens":attributes[SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = (value@@ -277,6 +281,9 @@ def process_chunk(scope, chunk):scope._output_tokens = (getattr(usage_metadata, "candidates_token_count", 0) if usage_metadata else 0)+ scope._reasoning_tokens = (+ getattr(usage_metadata, "thoughts_token_count", 0) if usage_metadata else 0+ ) or 0scope._cache_read_input_tokens = (getattr(usage_metadata, "cached_content_token_count", 0)if usage_metadata@@ -424,6 +431,17 @@ def common_chat_logic()scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)+ # Reasoning tokens+ if (+ hasattr(scope, "_reasoning_tokens")+ and scope._reasoning_tokens+ and scope._reasoning_tokens > 0+ ):+ scope._span.set_attribute(+ SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,+ scope._reasoning_tokens,+ )+# OTel cached token attributes (set even when 0)if hasattr(scope, "_cache_read_input_tokens"):scope._span.set_attribute(@@ -529,6 +547,8 @@ def common_chat_logic("output_tokens": output_tokens,**version_extras,}+ if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens:+ extra["reasoning_tokens"] = scope._reasoning_tokensif capture_message_content and system_instr:extra["system_instructions"] = system_instremit_inference_event(@@ -642,6 +662,9 @@ def process_chat_response(scope._output_tokens = (getattr(usage_metadata, "candidates_token_count", 0) if usage_metadata else 0)+ scope._reasoning_tokens = (+ getattr(usage_metadata, "thoughts_token_count", 0) if usage_metadata else 0+ ) or 0scope._cache_read_input_tokens = (getattr(usage_metadata, "cached_content_token_count", 0)if usage_metadatadiff --git a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.pyindex 93c337ce..366fdf35 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py@@ -59,6 +59,7 @@ def send_message(self._llmresponse = ""self._input_tokens = 0self._output_tokens = 0+ self._reasoning_tokens = 0self._cache_read_input_tokens = 0self._cache_creation_input_tokens = 0self._kwargs = kwargsdiff --git a/sdk/python/tests/test_vertexai_reasoning_tokens.py b/sdk/python/tests/test_vertexai_reasoning_tokens.pynew file mode 100644index 00000000..a6fcb40c--- /dev/null+++ b/sdk/python/tests/test_vertexai_reasoning_tokens.py@@ -0,0 +1,253 @@+# pylint: disable=protected-access, missing-function-docstring+"""Vertex AI thoughts_token_count is not a subset of output tokens.++Gemini/Vertex report thinking separately from candidate output:++ * ``candidates_token_count`` = visible output only+ * ``thoughts_token_count`` = thinking tokens (an addend to total, not a+ facet of output)+ * ``total_token_count`` = prompt + candidates + thoughts (+ tool-use)++That is the opposite of the OpenAI #1537 subset invariant, where reasoning+is already inside ``output_tokens``. These tests lock the Vertex instrumentor+to the google_ai_studio pattern: emit ``gen_ai.usage.reasoning_tokens`` when+thoughts > 0, and leave ``gen_ai.usage.output_tokens`` as candidates only.+"""++import time+from types import SimpleNamespace+from unittest.mock import MagicMock++from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.vertexai import utils as vertexai_utils+from openlit.semcov import SemanticConvention++PROMPT_TOKENS = 100+CANDIDATE_TOKENS = 50+THOUGHT_TOKENS = 25+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _usage(prompt=PROMPT_TOKENS, candidates=CANDIDATE_TOKENS, thoughts=None):+ kwargs = {+ "prompt_token_count": prompt,+ "candidates_token_count": candidates,+ "cached_content_token_count": 0,+ "cache_creation_input_tokens": 0,+ }+ if thoughts is not None:+ kwargs["thoughts_token_count"] = thoughts+ return SimpleNamespace(**kwargs)+++def _candidate(finish_reason="STOP"):+ return SimpleNamespace(finish_reason=finish_reason)+++def _response(usage, text="hello"):+ return SimpleNamespace(+ text=text,+ usage_metadata=usage,+ candidates=[_candidate()],+ id="resp_vertex_1",+ name=None,+ )+++def _stream_scope(span):+ """Mirrors TracedSyncStream.__init__ plus fields process_chunk writes."""+ return SimpleNamespace(+ _span=span,+ _llmresponse="",+ _finish_reason="",+ _response_id="",+ _input_tokens=0,+ _output_tokens=0,+ _reasoning_tokens=0,+ _cache_read_input_tokens=0,+ _cache_creation_input_tokens=0,+ _response_model="gemini-2.5-pro",+ _request_model="gemini-2.5-pro",+ _tools=None,+ _kwargs={"contents": [], "generation_config": {}},+ _args=[[]],+ _start_time=time.time(),+ _end_time=None,+ _timestamps=[],+ _ttft=0,+ _tbt=0,+ _server_address="us-central1-aiplatform.googleapis.com",+ _server_port=443,+ )+++def _chunk(text="", usage=None, finish_reason=""):+ return SimpleNamespace(+ text=text,+ usage_metadata=usage,+ candidates=[_candidate(finish_reason)] if finish_reason else [],+ )+++def _assert_thoughts_not_folded_into_output(attrs, thoughts=THOUGHT_TOKENS):+ """Thoughts are recorded separately and must not change output_tokens."""+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == PROMPT_TOKENS+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == CANDIDATE_TOKENS+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == thoughts+ # Total usage stays prompt + candidates, matching google_ai_studio.+ assert (+ attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]+ == PROMPT_TOKENS + CANDIDATE_TOKENS+ )+ # OpenAI #1537 subset attributes must not be applied to Vertex/Gemini.+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs+++def test_non_streaming_records_thoughts_separately_from_candidates():+ tracer, exporter = _tracer_with_exporter()+ response = _response(_usage(thoughts=THOUGHT_TOKENS))+ with tracer.start_as_current_span("vertexai.chat") as span:+ vertexai_utils.process_chat_response(+ response=response,+ request_model="gemini-2.5-pro",+ pricing_info={},+ server_port=443,+ server_address="us-central1-aiplatform.googleapis.com",+ environment="test",+ application_name="test",+ metrics=None,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=True,+ version="1.0.0",+ contents=[],+ )+ attrs = exporter.get_finished_spans()[0].attributes+ _assert_thoughts_not_folded_into_output(attrs)+++def test_streaming_last_chunk_usage_records_thoughts():+ """Vertex streams usage_metadata on the last chunk only."""+ tracer, exporter = _tracer_with_exporter()+ with tracer.start_as_current_span("vertexai.chat") as span:+ scope = _stream_scope(span)+ vertexai_utils.process_chunk(scope, _chunk(text="hel"))+ vertexai_utils.process_chunk(+ scope,+ _chunk(+ text="lo",+ usage=_usage(thoughts=THOUGHT_TOKENS),+ finish_reason="STOP",+ ),+ )+ vertexai_utils.process_streaming_chat_response(+ scope,+ pricing_info={},+ environment="test",+ application_name="test",+ metrics=None,+ capture_message_content=False,+ disable_metrics=True,+ version="1.0.0",+ )+ attrs = exporter.get_finished_spans()[0].attributes+ _assert_thoughts_not_folded_into_output(attrs)+++def test_missing_thoughts_does_not_emit_reasoning_attribute():+ tracer, exporter = _tracer_with_exporter()+ response = _response(_usage(thoughts=None))+ with tracer.start_as_current_span("vertexai.chat") as span:+ vertexai_utils.process_chat_response(+ response=response,+ request_model="gemini-2.5-pro",+ pricing_info={},+ server_port=443,+ server_address="us-central1-aiplatform.googleapis.com",+ environment="test",+ application_name="test",+ metrics=None,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=True,+ version="1.0.0",+ contents=[],+ )+ attrs = exporter.get_finished_spans()[0].attributes+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == CANDIDATE_TOKENS+++def test_zero_thoughts_does_not_emit_reasoning_attribute():+ tracer, exporter = _tracer_with_exporter()+ response = _response(_usage(thoughts=0))+ with tracer.start_as_current_span("vertexai.chat") as span:+ vertexai_utils.process_chat_response(+ response=response,+ request_model="gemini-2.5-pro",+ pricing_info={},+ server_port=443,+ server_address="us-central1-aiplatform.googleapis.com",+ environment="test",+ application_name="test",+ metrics=None,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=True,+ version="1.0.0",+ contents=[],+ )+ attrs = exporter.get_finished_spans()[0].attributes+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs+++def test_inference_event_includes_reasoning_tokens():+ tracer, exporter = _tracer_with_exporter()+ event_provider = MagicMock()+ response = _response(_usage(thoughts=THOUGHT_TOKENS))+ with tracer.start_as_current_span("vertexai.chat") as span:+ vertexai_utils.process_chat_response(+ response=response,+ request_model="gemini-2.5-pro",+ pricing_info={},+ server_port=443,+ server_address="us-central1-aiplatform.googleapis.com",+ environment="test",+ application_name="test",+ metrics=None,+ start_time=time.time(),+ span=span,+ capture_message_content=False,+ disable_metrics=True,+ version="1.0.0",+ event_provider=event_provider,+ contents=[],+ )+ exporter.get_finished_spans()+ event_provider.emit.assert_called_once()+ event = event_provider.emit.call_args.args[0]+ assert (+ event.attributes[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS]+ == THOUGHT_TOKENS+ )+ assert (+ event.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]+ == CANDIDATE_TOKENS+ )