Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(vertexai): track reasoning tokens from thoughts_token_count (#1544)

Goutham Annem9cbcc324e8bc
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
index 370d0c6b..e9f60e89 100644
--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
@@ -59,6 +59,7 @@ def async_send_message(
self._llmresponse = ""
self._input_tokens = 0
self._output_tokens = 0
+ self._reasoning_tokens = 0
self._cache_read_input_tokens = 0
self._cache_creation_input_tokens = 0
self._kwargs = kwargs
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/utils.py b/sdk/python/src/openlit/instrumentation/vertexai/utils.py
index 1e6b0cec..9077eb40 100644
--- a/sdk/python/src/openlit/instrumentation/vertexai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/vertexai/utils.py
@@ -229,6 +229,10 @@ def emit_inference_event(
attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = value
elif key == "output_tokens":
attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = value
+ elif key == "reasoning_tokens":
+ # Vertex/Gemini: thoughts_token_count is separate from
+ # candidates_token_count, not a subset of output tokens.
+ attributes[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] = value
elif key == "cache_read_input_tokens":
attributes[SemanticConvention.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = (
value
@@ -277,6 +281,9 @@ def process_chunk(scope, chunk):
scope._output_tokens = (
getattr(usage_metadata, "candidates_token_count", 0) if usage_metadata else 0
)
+ scope._reasoning_tokens = (
+ getattr(usage_metadata, "thoughts_token_count", 0) if usage_metadata else 0
+ ) or 0
scope._cache_read_input_tokens = (
getattr(usage_metadata, "cached_content_token_count", 0)
if usage_metadata
@@ -424,6 +431,17 @@ def common_chat_logic(
)
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
+ # Reasoning tokens
+ if (
+ hasattr(scope, "_reasoning_tokens")
+ and scope._reasoning_tokens
+ and scope._reasoning_tokens > 0
+ ):
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS,
+ scope._reasoning_tokens,
+ )
+
# OTel cached token attributes (set even when 0)
if hasattr(scope, "_cache_read_input_tokens"):
scope._span.set_attribute(
@@ -529,6 +547,8 @@ def common_chat_logic(
"output_tokens": output_tokens,
**version_extras,
}
+ if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens:
+ extra["reasoning_tokens"] = scope._reasoning_tokens
if capture_message_content and system_instr:
extra["system_instructions"] = system_instr
emit_inference_event(
@@ -642,6 +662,9 @@ def process_chat_response(
scope._output_tokens = (
getattr(usage_metadata, "candidates_token_count", 0) if usage_metadata else 0
)
+ scope._reasoning_tokens = (
+ getattr(usage_metadata, "thoughts_token_count", 0) if usage_metadata else 0
+ ) or 0
scope._cache_read_input_tokens = (
getattr(usage_metadata, "cached_content_token_count", 0)
if usage_metadata
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py
index 93c337ce..366fdf35 100644
--- a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py
+++ b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py
@@ -59,6 +59,7 @@ def send_message(
self._llmresponse = ""
self._input_tokens = 0
self._output_tokens = 0
+ self._reasoning_tokens = 0
self._cache_read_input_tokens = 0
self._cache_creation_input_tokens = 0
self._kwargs = kwargs
diff --git a/sdk/python/tests/test_vertexai_reasoning_tokens.py b/sdk/python/tests/test_vertexai_reasoning_tokens.py
new file mode 100644
index 00000000..a6fcb40c
--- /dev/null
+++ b/sdk/python/tests/test_vertexai_reasoning_tokens.py
@@ -0,0 +1,253 @@
+# pylint: disable=protected-access, missing-function-docstring
+"""Vertex AI thoughts_token_count is not a subset of output tokens.
+
+Gemini/Vertex report thinking separately from candidate output:
+
+ * ``candidates_token_count`` = visible output only
+ * ``thoughts_token_count`` = thinking tokens (an addend to total, not a
+ facet of output)
+ * ``total_token_count`` = prompt + candidates + thoughts (+ tool-use)
+
+That is the opposite of the OpenAI #1537 subset invariant, where reasoning
+is already inside ``output_tokens``. These tests lock the Vertex instrumentor
+to the google_ai_studio pattern: emit ``gen_ai.usage.reasoning_tokens`` when
+thoughts > 0, and leave ``gen_ai.usage.output_tokens`` as candidates only.
+"""
+
+import time
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.vertexai import utils as vertexai_utils
+from openlit.semcov import SemanticConvention
+
+PROMPT_TOKENS = 100
+CANDIDATE_TOKENS = 50
+THOUGHT_TOKENS = 25
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _usage(prompt=PROMPT_TOKENS, candidates=CANDIDATE_TOKENS, thoughts=None):
+ kwargs = {
+ "prompt_token_count": prompt,
+ "candidates_token_count": candidates,
+ "cached_content_token_count": 0,
+ "cache_creation_input_tokens": 0,
+ }
+ if thoughts is not None:
+ kwargs["thoughts_token_count"] = thoughts
+ return SimpleNamespace(**kwargs)
+
+
+def _candidate(finish_reason="STOP"):
+ return SimpleNamespace(finish_reason=finish_reason)
+
+
+def _response(usage, text="hello"):
+ return SimpleNamespace(
+ text=text,
+ usage_metadata=usage,
+ candidates=[_candidate()],
+ id="resp_vertex_1",
+ name=None,
+ )
+
+
+def _stream_scope(span):
+ """Mirrors TracedSyncStream.__init__ plus fields process_chunk writes."""
+ return SimpleNamespace(
+ _span=span,
+ _llmresponse="",
+ _finish_reason="",
+ _response_id="",
+ _input_tokens=0,
+ _output_tokens=0,
+ _reasoning_tokens=0,
+ _cache_read_input_tokens=0,
+ _cache_creation_input_tokens=0,
+ _response_model="gemini-2.5-pro",
+ _request_model="gemini-2.5-pro",
+ _tools=None,
+ _kwargs={"contents": [], "generation_config": {}},
+ _args=[[]],
+ _start_time=time.time(),
+ _end_time=None,
+ _timestamps=[],
+ _ttft=0,
+ _tbt=0,
+ _server_address="us-central1-aiplatform.googleapis.com",
+ _server_port=443,
+ )
+
+
+def _chunk(text="", usage=None, finish_reason=""):
+ return SimpleNamespace(
+ text=text,
+ usage_metadata=usage,
+ candidates=[_candidate(finish_reason)] if finish_reason else [],
+ )
+
+
+def _assert_thoughts_not_folded_into_output(attrs, thoughts=THOUGHT_TOKENS):
+ """Thoughts are recorded separately and must not change output_tokens."""
+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == PROMPT_TOKENS
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == CANDIDATE_TOKENS
+ assert attrs[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS] == thoughts
+ # Total usage stays prompt + candidates, matching google_ai_studio.
+ assert (
+ attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]
+ == PROMPT_TOKENS + CANDIDATE_TOKENS
+ )
+ # OpenAI #1537 subset attributes must not be applied to Vertex/Gemini.
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_DERIVED_COMPLETED_OUTPUT_TOKENS not in attrs
+
+
+def test_non_streaming_records_thoughts_separately_from_candidates():
+ tracer, exporter = _tracer_with_exporter()
+ response = _response(_usage(thoughts=THOUGHT_TOKENS))
+ with tracer.start_as_current_span("vertexai.chat") as span:
+ vertexai_utils.process_chat_response(
+ response=response,
+ request_model="gemini-2.5-pro",
+ pricing_info={},
+ server_port=443,
+ server_address="us-central1-aiplatform.googleapis.com",
+ environment="test",
+ application_name="test",
+ metrics=None,
+ start_time=time.time(),
+ span=span,
+ capture_message_content=False,
+ disable_metrics=True,
+ version="1.0.0",
+ contents=[],
+ )
+ attrs = exporter.get_finished_spans()[0].attributes
+ _assert_thoughts_not_folded_into_output(attrs)
+
+
+def test_streaming_last_chunk_usage_records_thoughts():
+ """Vertex streams usage_metadata on the last chunk only."""
+ tracer, exporter = _tracer_with_exporter()
+ with tracer.start_as_current_span("vertexai.chat") as span:
+ scope = _stream_scope(span)
+ vertexai_utils.process_chunk(scope, _chunk(text="hel"))
+ vertexai_utils.process_chunk(
+ scope,
+ _chunk(
+ text="lo",
+ usage=_usage(thoughts=THOUGHT_TOKENS),
+ finish_reason="STOP",
+ ),
+ )
+ vertexai_utils.process_streaming_chat_response(
+ scope,
+ pricing_info={},
+ environment="test",
+ application_name="test",
+ metrics=None,
+ capture_message_content=False,
+ disable_metrics=True,
+ version="1.0.0",
+ )
+ attrs = exporter.get_finished_spans()[0].attributes
+ _assert_thoughts_not_folded_into_output(attrs)
+
+
+def test_missing_thoughts_does_not_emit_reasoning_attribute():
+ tracer, exporter = _tracer_with_exporter()
+ response = _response(_usage(thoughts=None))
+ with tracer.start_as_current_span("vertexai.chat") as span:
+ vertexai_utils.process_chat_response(
+ response=response,
+ request_model="gemini-2.5-pro",
+ pricing_info={},
+ server_port=443,
+ server_address="us-central1-aiplatform.googleapis.com",
+ environment="test",
+ application_name="test",
+ metrics=None,
+ start_time=time.time(),
+ span=span,
+ capture_message_content=False,
+ disable_metrics=True,
+ version="1.0.0",
+ contents=[],
+ )
+ attrs = exporter.get_finished_spans()[0].attributes
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == CANDIDATE_TOKENS
+
+
+def test_zero_thoughts_does_not_emit_reasoning_attribute():
+ tracer, exporter = _tracer_with_exporter()
+ response = _response(_usage(thoughts=0))
+ with tracer.start_as_current_span("vertexai.chat") as span:
+ vertexai_utils.process_chat_response(
+ response=response,
+ request_model="gemini-2.5-pro",
+ pricing_info={},
+ server_port=443,
+ server_address="us-central1-aiplatform.googleapis.com",
+ environment="test",
+ application_name="test",
+ metrics=None,
+ start_time=time.time(),
+ span=span,
+ capture_message_content=False,
+ disable_metrics=True,
+ version="1.0.0",
+ contents=[],
+ )
+ attrs = exporter.get_finished_spans()[0].attributes
+ assert SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS not in attrs
+
+
+def test_inference_event_includes_reasoning_tokens():
+ tracer, exporter = _tracer_with_exporter()
+ event_provider = MagicMock()
+ response = _response(_usage(thoughts=THOUGHT_TOKENS))
+ with tracer.start_as_current_span("vertexai.chat") as span:
+ vertexai_utils.process_chat_response(
+ response=response,
+ request_model="gemini-2.5-pro",
+ pricing_info={},
+ server_port=443,
+ server_address="us-central1-aiplatform.googleapis.com",
+ environment="test",
+ application_name="test",
+ metrics=None,
+ start_time=time.time(),
+ span=span,
+ capture_message_content=False,
+ disable_metrics=True,
+ version="1.0.0",
+ event_provider=event_provider,
+ contents=[],
+ )
+ exporter.get_finished_spans()
+ event_provider.emit.assert_called_once()
+ event = event_provider.emit.call_args.args[0]
+ assert (
+ event.attributes[SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS]
+ == THOUGHT_TOKENS
+ )
+ assert (
+ event.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]
+ == CANDIDATE_TOKENS
+ )