Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Refactor VertexAI OpenTelemetry Monitoring Instrumentation (#784)
Patcher9560aaa5b759
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 4dde472b..b1fa22db 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.34.20"+version = "1.34.21"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/vertexai/__init__.py b/sdk/python/src/openlit/instrumentation/vertexai/__init__.pyindex a7ba0b94..5e944282 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/__init__.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/__init__.py@@ -1,4 +1,3 @@-# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code"""Initializer of Auto Instrumentation of VertexAI Functions"""from typing import Collection@@ -6,92 +5,88 @@ import importlib.metadatafrom opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapper-from openlit.instrumentation.vertexai.vertexai import (- send_message-)-from openlit.instrumentation.vertexai.async_vertexai import (- async_send_message-)-+from openlit.instrumentation.vertexai.vertexai import send_message+from openlit.instrumentation.vertexai.async_vertexai import async_send_message_instruments = ("google-cloud-aiplatform >= 1.38.1",)class VertexAIInstrumentor(BaseInstrumentor):"""- An instrumentor for VertexAI's client library.+ An instrumentor for VertexAI client library."""def instrumentation_dependencies(self) -> Collection[str]:return _instrumentsdef _instrument(self, **kwargs):- application_name = kwargs.get("application_name", "default")+ version = importlib.metadata.version("google-cloud-aiplatform")environment = kwargs.get("environment", "default")+ application_name = kwargs.get("application_name", "default")tracer = kwargs.get("tracer")- metrics = kwargs.get("metrics_dict")pricing_info = kwargs.get("pricing_info", {})capture_message_content = kwargs.get("capture_message_content", False)+ metrics = kwargs.get("metrics_dict")disable_metrics = kwargs.get("disable_metrics")- version = importlib.metadata.version("google-cloud-aiplatform")- #sync+ # sync generative modelswrap_function_wrapper("vertexai.generative_models","GenerativeModel.generate_content",send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.generative_models","ChatSession.send_message",send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)+ # sync language modelswrap_function_wrapper("vertexai.language_models","ChatSession.send_message",send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message_streaming",send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)- #async+ # async generative modelswrap_function_wrapper("vertexai.generative_models","GenerativeModel.generate_content_async",async_send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.generative_models","ChatSession.send_message_async",async_send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)+ # async language modelswrap_function_wrapper("vertexai.language_models","ChatSession.send_message_async",async_send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message_streaming_async",async_send_message(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)def _uninstrument(self, **kwargs):- # Proper uninstrumentation logic to revert patched methodspassdiff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.pyindex ca2b05a9..9c2e4892 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py@@ -4,14 +4,15 @@ Module for monitoring VertexAI API calls.import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,handle_exception,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,+)+from openlit.instrumentation.vertexai.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,+ extract_vertexai_details,)from openlit.semcov import SemanticConvention@@ -21,35 +22,19 @@ logger = logging.getLogger(__name__)def async_send_message(version, environment, application_name, tracer,pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for messages to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the VertexAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of VertexAI usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for VertexAI AsyncMessages calls."""class TracedAsyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for async streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -58,12 +43,10 @@ def async_send_message(version, environment, application_name, tracer,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""- self._input_tokens = ""- self._output_tokens = ""-- self._args = args+ self._input_tokens = 0+ self._output_tokens = 0self._kwargs = kwargsself._start_time = time.time()self._end_time = None@@ -73,6 +56,7 @@ def async_send_message(version, environment, application_name, tracer,self._server_address = server_addressself._server_port = server_portself._request_model = request_model+ self._args = argsasync def __aenter__(self):await self.__wrapped__.__aenter__()@@ -91,369 +75,67 @@ def async_send_message(version, environment, application_name, tracer,async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- self._llmresponse += str(chunk.text)- self._input_tokens = chunk.usage_metadata.prompt_token_count- self._output_tokens = chunk.usage_metadata.candidates_token_count-+ process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages) or str(self._args[0][0])-- # Calculate cost of the operation- cost = get_chat_model_cost(self._request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_VERTEXAI)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- self._request_model)- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)-- inference_config = self._kwargs.get('generation_config', {})-- # List of attributes and their config keys- attributes = [- (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),- (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),- (SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),- (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),- (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),- (SemanticConvention.GEN_AI_REQUEST_TOP_P, 'top_p'),- (SemanticConvention.GEN_AI_REQUEST_TOP_K, 'top_k'),- ]-- # Set each attribute if the corresponding value exists and is not None- for attribute, key in attributes:- # Use the `get` method to safely access keys in the dictionary- value = inference_config.get(key)- if value is not None:- self._span.set_attribute(attribute, value)-- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- self._request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,- request_model=self._request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._request_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-except Exception as e:handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'messages' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'messages' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'messages' method.- kwargs: Keyword arguments for the 'messages' method.-- Returns:- The response from the original 'messages' method.+ Wraps the VertexAI async API call to add telemetry."""- # Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)-- try:- location = instance._model._location- request_model = "/".join(instance._model._model_name.split("/")[3:])- except:- location = instance._location- request_model = "/".join(instance._model_name.split("/")[3:])-- server_address, server_port = location + '-aiplatform.googleapis.com', 443+ server_address, server_port, request_model = extract_vertexai_details(instance)span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = await wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port, request_model, args)+ return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port, request_model, args)- # Handling for non-streaming responseselse:with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = await wrapped(*args, **kwargs)- end_time = time.time()try:- # Format 'messages' into a single string- message_prompt = kwargs.get("contents", [])- formatted_messages = []-- for content in message_prompt:- role = content.role- parts = content.parts- content_str = []-- for part in parts:- # Collect relevant fields and handle each type of data that Part could contain- if part.text:- content_str.append(f"text: {part.text}")- if part.video_metadata:- content_str.append(f"video_metadata: {part.video_metadata}")- if part.thought:- content_str.append(f"thought: {part.thought}")- if part.code_execution_result:- content_str.append(f"code_execution_result: {part.code_execution_result}")- if part.executable_code:- content_str.append(f"executable_code: {part.executable_code}")- if part.file_data:- content_str.append(f"file_data: {part.file_data}")- if part.function_call:- content_str.append(f"function_call: {part.function_call}")- if part.function_response:- content_str.append(f"function_response: {part.function_response}")- if part.inline_data:- content_str.append(f"inline_data: {part.inline_data}")-- formatted_messages.append(f"{role}: {', '.join(content_str)}")-- prompt = "\n".join(formatted_messages) or str(args[0][0])-- input_tokens = response.usage_metadata.prompt_token_count- output_tokens = response.usage_metadata.candidates_token_count-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)-- inference_config = kwargs.get('generation_config', {})-- # List of attributes and their config keys- attributes = [- (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),- (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),- (SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),- (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),- (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),- (SemanticConvention.GEN_AI_REQUEST_TOP_P, 'top_p'),- (SemanticConvention.GEN_AI_REQUEST_TOP_K, 'top_k'),- ]-- # Set each attribute if the corresponding value exists and is not None- for attribute, key in attributes:- # Use the `get` method to safely access keys in the dictionary- value = inference_config.get(key)- if value is not None:- span.set_attribute(attribute, value)-- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- # span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- # [str(response.candidates[0].finish_reason)])-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- if isinstance(response.text, str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- elif response.text is not None:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )except Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)- # Return original response- return response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/vertexai/utils.py b/sdk/python/src/openlit/instrumentation/vertexai/utils.pynew file mode 100644index 00000000..de1328b6--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/vertexai/utils.py@@ -0,0 +1,204 @@+"""+VertexAI OpenTelemetry instrumentation utility functions+"""+import time++from opentelemetry.trace import Status, StatusCode++from openlit.__helpers import (+ calculate_ttft,+ calculate_tbt,+ get_chat_model_cost,+ record_completion_metrics,+ common_span_attributes,+)+from openlit.semcov import SemanticConvention++def format_content(contents):+ """+ Format the VertexAI contents into a string for span events.+ """++ if not contents:+ return ""++ formatted_messages = []+ for content in contents:+ role = content.role+ parts = content.parts+ content_str = []++ for part in parts:+ # Collect relevant fields and handle each type of data that Part could contain+ if part.text:+ content_str.append(f"text: {part.text}")+ if part.video_metadata:+ content_str.append(f"video_metadata: {part.video_metadata}")+ if part.thought:+ content_str.append(f"thought: {part.thought}")+ if part.code_execution_result:+ content_str.append(f"code_execution_result: {part.code_execution_result}")+ if part.executable_code:+ content_str.append(f"executable_code: {part.executable_code}")+ if part.file_data:+ content_str.append(f"file_data: {part.file_data}")+ if part.function_call:+ content_str.append(f"function_call: {part.function_call}")+ if part.function_response:+ content_str.append(f"function_response: {part.function_response}")+ if part.inline_data:+ content_str.append(f"inline_data: {part.inline_data}")++ formatted_messages.append(f"{role}: {', '.join(content_str)}")++ return "\n".join(formatted_messages)++def process_chunk(scope, chunk):+ """+ Process a chunk of response data and update state.+ """++ end_time = time.time()+ # Record the timestamp for the current chunk+ scope._timestamps.append(end_time)++ if len(scope._timestamps) == 1:+ # Calculate time to first chunk+ scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)++ # Aggregate response content+ scope._llmresponse += str(chunk.text)+ scope._input_tokens = chunk.usage_metadata.prompt_token_count+ scope._output_tokens = chunk.usage_metadata.candidates_token_count++def common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream):+ """+ Process chat request and generate Telemetry+ """++ scope._end_time = time.time()+ if len(scope._timestamps) > 1:+ scope._tbt = calculate_tbt(scope._timestamps)++ # Format content using VertexAI-specific logic+ contents = scope._kwargs.get("contents", [])+ formatted_messages = format_content(contents)+ prompt = formatted_messages or str(scope._args[0][0])++ cost = get_chat_model_cost(scope._request_model, pricing_info, scope._input_tokens, scope._output_tokens)++ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,+ scope._server_address, scope._server_port, scope._request_model, scope._request_model,+ environment, application_name, is_stream, scope._tbt, scope._ttft, version)++ # Span Attributes for Request parameters (VertexAI-specific)+ inference_config = scope._kwargs.get("generation_config", {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, "frequency_penalty"),+ (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, "max_output_tokens"),+ (SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, "presence_penalty"),+ (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, "stop_sequences"),+ (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, "temperature"),+ (SemanticConvention.GEN_AI_REQUEST_TOP_P, "top_p"),+ (SemanticConvention.GEN_AI_REQUEST_TOP_K, "top_k"),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ value = inference_config.get(key)+ if value is not None:+ scope._span.set_attribute(attribute, value)++ # Span Attributes for Response parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text" if isinstance(scope._llmresponse, str) else "json")++ # Span Attributes for Cost and Tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)++ # Span Attributes for Content+ if capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse)++ # To be removed once the change to span_attributes (from span events) is complete+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,+ },+ )++ scope._span.set_status(Status(StatusCode.OK))++ # Record metrics+ if not disable_metrics:+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,+ scope._server_address, scope._server_port, scope._request_model, scope._request_model, environment,+ application_name, scope._start_time, scope._end_time, scope._input_tokens, scope._output_tokens,+ cost, scope._tbt, scope._ttft)++def process_streaming_chat_response(scope, pricing_info, environment, application_name, metrics,+ capture_message_content=False, disable_metrics=False, version=""):+ """+ Process streaming chat response and generate telemetry.+ """++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=True)++def process_chat_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, start_time,+ span, capture_message_content=False, disable_metrics=False, version="1.0.0", **kwargs):+ """+ Process non-streaming chat response and generate telemetry.+ """++ scope = type("GenericScope", (), {})()++ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ scope._llmresponse = response.text+ scope._input_tokens = response.usage_metadata.prompt_token_count+ scope._output_tokens = response.usage_metadata.candidates_token_count+ scope._timestamps = []+ scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0+ scope._server_address, scope._server_port = server_address, server_port+ scope._request_model = request_model+ scope._kwargs = kwargs+ scope._args = [kwargs.get("contents", [])]++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=False)++ return response++def extract_vertexai_details(instance):+ """+ Extract VertexAI-specific details like location and model name.+ """+ try:+ location = instance._model._location+ request_model = "/".join(instance._model._model_name.split("/")[3:])+ except:+ location = instance._location+ request_model = "/".join(instance._model_name.split("/")[3:])++ server_address = location + "-aiplatform.googleapis.com"+ server_port = 443++ return server_address, server_port, request_modeldiff --git a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.pyindex 811c6729..905ca68d 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py@@ -4,14 +4,15 @@ Module for monitoring VertexAI API calls.import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,handle_exception,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,+)+from openlit.instrumentation.vertexai.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,+ extract_vertexai_details,)from openlit.semcov import SemanticConvention@@ -21,35 +22,19 @@ logger = logging.getLogger(__name__)def send_message(version, environment, application_name, tracer,pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for messages to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the VertexAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of VertexAI usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for VertexAI messages to collect metrics."""class TracedSyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -58,12 +43,10 @@ def send_message(version, environment, application_name, tracer,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""- self._input_tokens = ""- self._output_tokens = ""-- self._args = args+ self._input_tokens = 0+ self._output_tokens = 0self._kwargs = kwargsself._start_time = time.time()self._end_time = None@@ -73,6 +56,7 @@ def send_message(version, environment, application_name, tracer,self._server_address = server_addressself._server_port = server_portself._request_model = request_model+ self._args = argsdef __enter__(self):self.__wrapped__.__enter__()@@ -91,369 +75,67 @@ def send_message(version, environment, application_name, tracer,def __next__(self):try:chunk = self.__wrapped__.__next__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- self._llmresponse += str(chunk.text)- self._input_tokens = chunk.usage_metadata.prompt_token_count- self._output_tokens = chunk.usage_metadata.candidates_token_count-+ process_chunk(self, chunk)return chunkexcept StopIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages) or str(self._args[0][0])-- # Calculate cost of the operation- cost = get_chat_model_cost(self._request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_VERTEXAI)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- self._request_model)- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)-- inference_config = self._kwargs.get('generation_config', {})-- # List of attributes and their config keys- attributes = [- (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),- (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),- (SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),- (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),- (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),- (SemanticConvention.GEN_AI_REQUEST_TOP_P, 'top_p'),- (SemanticConvention.GEN_AI_REQUEST_TOP_K, 'top_k'),- ]-- # Set each attribute if the corresponding value exists and is not None- for attribute, key in attributes:- # Use the `get` method to safely access keys in the dictionary- value = inference_config.get(key)- if value is not None:- self._span.set_attribute(attribute, value)-- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- self._request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,- request_model=self._request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._request_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-except Exception as e:handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'messages' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'messages' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'messages' method.- kwargs: Keyword arguments for the 'messages' method.-- Returns:- The response from the original 'messages' method.+ Wraps the VertexAI API call to add telemetry."""- # Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)-- try:- location = instance._model._location- request_model = "/".join(instance._model._model_name.split("/")[3:])- except:- location = instance._location- request_model = "/".join(instance._model_name.split("/")[3:])-- server_address, server_port = location + '-aiplatform.googleapis.com', 443+ server_address, server_port, request_model = extract_vertexai_details(instance)span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port, request_model, args)+ return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port, request_model, args)- # Handling for non-streaming responseselse:with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = wrapped(*args, **kwargs)- end_time = time.time()try:- # Format 'messages' into a single string- message_prompt = kwargs.get("contents", [])- formatted_messages = []-- for content in message_prompt:- role = content.role- parts = content.parts- content_str = []-- for part in parts:- # Collect relevant fields and handle each type of data that Part could contain- if part.text:- content_str.append(f"text: {part.text}")- if part.video_metadata:- content_str.append(f"video_metadata: {part.video_metadata}")- if part.thought:- content_str.append(f"thought: {part.thought}")- if part.code_execution_result:- content_str.append(f"code_execution_result: {part.code_execution_result}")- if part.executable_code:- content_str.append(f"executable_code: {part.executable_code}")- if part.file_data:- content_str.append(f"file_data: {part.file_data}")- if part.function_call:- content_str.append(f"function_call: {part.function_call}")- if part.function_response:- content_str.append(f"function_response: {part.function_response}")- if part.inline_data:- content_str.append(f"inline_data: {part.inline_data}")-- formatted_messages.append(f"{role}: {', '.join(content_str)}")-- prompt = "\n".join(formatted_messages) or str(args[0][0])-- input_tokens = response.usage_metadata.prompt_token_count- output_tokens = response.usage_metadata.candidates_token_count-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)-- inference_config = kwargs.get('generation_config', {})-- # List of attributes and their config keys- attributes = [- (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),- (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),- (SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),- (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),- (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),- (SemanticConvention.GEN_AI_REQUEST_TOP_P, 'top_p'),- (SemanticConvention.GEN_AI_REQUEST_TOP_K, 'top_k'),- ]-- # Set each attribute if the corresponding value exists and is not None- for attribute, key in attributes:- # Use the `get` method to safely access keys in the dictionary- value = inference_config.get(key)- if value is not None:- span.set_attribute(attribute, value)-- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- # span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- # [str(response.candidates[0].finish_reason)])-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- if isinstance(response.text, str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- elif response.text is not None:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_VERTEXAI,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )except Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)- # Return original response- return response+ return responsereturn wrapper