Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update vLLM and VertexAI OpenTelemetry Instrumentations (#632)
Patcher780c10d77765
diff --git a/sdk/python/.pylintrc b/sdk/python/.pylintrcindex fe2409c9..3bfcf9bd 100644--- a/sdk/python/.pylintrc+++ b/sdk/python/.pylintrc@@ -26,4 +26,5 @@ disable=too-many-locals,too-many-lines,no-name-in-module,- protected-access,\ No newline at end of file+ protected-access,+ bare-except\ No newline at end of filediff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex eea7ee41..dde0383f 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.10"+version = "1.33.9"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/vertexai/__init__.py b/sdk/python/src/openlit/instrumentation/vertexai/__init__.pyindex f89a4d0b..24901f68 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/__init__.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/__init__.py@@ -7,16 +7,10 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapperfrom openlit.instrumentation.vertexai.vertexai import (- generate_content, predict, predict_streaming,- send_message, start_chat, start_chat_streaming,- embeddings+ send_message)from openlit.instrumentation.vertexai.async_vertexai import (- generate_content_async, predict_async,- predict_streaming_async,- send_message_async,- start_chat_async, start_chat_streaming_async,- embeddings_async+ async_send_message)@@ -44,49 +38,28 @@ class VertexAIInstrumentor(BaseInstrumentor):wrap_function_wrapper("vertexai.generative_models","GenerativeModel.generate_content",- generate_content("vertexai.generate_content", version, environment, application_name,+ send_message(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.generative_models","ChatSession.send_message",- send_message("vertexai.send_message", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextGenerationModel.predict",- predict("vertexai.predict", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextGenerationModel.predict_streaming",- predict_streaming("vertexai.predict", version, environment, application_name,+ send_message(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message",- start_chat("vertexai.send_message", version, environment, application_name,+ send_message(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message_streaming",- start_chat_streaming("vertexai.send_message", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextEmbeddingModel.get_embeddings",- embeddings("vertexai.get_embeddings", version, environment, application_name,+ send_message(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -94,52 +67,29 @@ class VertexAIInstrumentor(BaseInstrumentor):wrap_function_wrapper("vertexai.generative_models","GenerativeModel.generate_content_async",- generate_content_async("vertexai.generate_content", version, environment,- application_name, tracer, pricing_info, trace_content,- metrics, disable_metrics),+ async_send_message(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.generative_models","ChatSession.send_message_async",- send_message_async("vertexai.send_message", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextGenerationModel.predict_async",- predict_async("vertexai.predict", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextGenerationModel.predict_streaming_async",- predict_streaming_async("vertexai.predict", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),+ async_send_message(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message_async",- start_chat_async("vertexai.send_message", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),+ async_send_message(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("vertexai.language_models","ChatSession.send_message_streaming_async",- start_chat_streaming_async("vertexai.send_message", version, environment,- application_name, tracer, pricing_info, trace_content,- metrics, disable_metrics),- )-- wrap_function_wrapper(- "vertexai.language_models",- "TextEmbeddingModel.get_embeddings_async",- embeddings_async("vertexai.get_embeddings", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),+ async_send_message(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),)def _uninstrument(self, **kwargs):diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.pyindex e4a406a0..e9900578 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py@@ -1,693 +1,402 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-lines"""Module for monitoring VertexAI API calls."""import logging-import math+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def generate_content_async(gen_ai_endpoint, version, environment, application_name, tracer,+def async_send_message(version, environment, application_name, tracer,pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for messages to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the VertexAI API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of VertexAI usage.trace_content: Flag indicating whether to trace the actual content.Returns:A function that wraps the chat method to add telemetry."""- async def wrapper(wrapped, instance, args, kwargs):+ class TracedAsyncStream:"""- Wraps the 'generate_content' API call to add telemetry.+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the response to collect message IDs and aggregated response.- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.+ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol."""+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ request_model,+ args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._input_tokens = ""+ self._output_tokens = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port+ self._request_model = request_model++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)-- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in await wrapped(*args, **kwargs):- llmresponse += str(event.text)- prompt_tokens = event.usage_metadata.prompt_token_count- completion_tokens = event.usage_metadata.candidates_token_count- total_tokens = event.usage_metadata.total_token_count- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = str(args[0][0])-- model = "/".join(instance._model_name.split("/")[3:])-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)- return stream_generator()-- # Handling for non-streaming responses- else:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)+ self._llmresponse += str(chunk.text)+ self._input_tokens = chunk.usage_metadata.prompt_token_count+ self._output_tokens = chunk.usage_metadata.candidates_token_count+ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single string- prompt = str(args[0][0])+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages) or str(self._args[0][0])- model = "/".join(instance._model_name.split("/")[3:])# Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info,- response.usage_metadata.prompt_token_count,- response.usage_metadata.candidates_token_count)+ cost = get_chat_model_cost(self._request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)++ inference_config = self._kwargs.get('generation_config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use the `get` method to safely access keys in the dictionary+ value = inference_config.get(key)+ if value is not None:+ self._span.set_attribute(attribute, value)++ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)- span.set_attribute(SERVICE_NAME,+ self._span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage_metadata.prompt_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage_metadata.candidates_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage_metadata.total_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.candidates[0].content.parts[0].text,+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-- span.set_status(Status(StatusCode.OK))+ self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,+ request_model=self._request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage_metadata.total_token_count, attributes)- metrics["genai_completion_tokens"].add(- response.usage_metadata.candidates_token_count, attributes)- metrics["genai_prompt_tokens"].add(- response.usage_metadata.prompt_token_count, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)- # Return original response- return response-except Exception as e:- handle_exception(span, e)+ handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper---def send_message_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """+ finally:+ self._span.end()+ raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'generate_content' API call to add telemetry.+ Wraps the 'messages' API call to add telemetry.This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'generate_content' method to be wrapped.+ wrapped: The original 'messages' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.+ args: Positional arguments for the 'messages' method.+ kwargs: Keyword arguments for the 'messages' method.Returns:- The response from the original 'generate_content' method.+ The response from the original 'messages' method."""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in await wrapped(*args, **kwargs):- llmresponse += str(event.text)- prompt_tokens = event.usage_metadata.prompt_token_count- completion_tokens = event.usage_metadata.candidates_token_count- total_tokens = event.usage_metadata.total_token_count- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0][0]-- model = "/".join(instance._model._model_name.split("/")[3:])-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ try:+ location = instance._model._location+ request_model = "/".join(instance._model._model_name.split("/")[3:])+ except:+ location = instance._location+ request_model = "/".join(instance._model_name.split("/")[3:])- span.set_status(Status(StatusCode.OK))+ server_address, server_port = location + '-aiplatform.googleapis.com', 443- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # pylint: disable=no-else-return+ if streaming:+ # Special handling for streaming response to accommodate the nature of data flow+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return stream_generator()+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port, request_model, args)# Handling for non-streaming responseselse:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()try:# Format 'messages' into a single string- prompt = args[0]+ message_prompt = kwargs.get("contents", [])+ formatted_messages = []++ for content in message_prompt:+ role = content.role+ parts = content.parts+ content_str = []++ for part in parts:+ # Collect relevant fields and handle each type of data that Part could contain+ if part.text:+ content_str.append(f"text: {part.text}")+ if part.video_metadata:+ content_str.append(f"video_metadata: {part.video_metadata}")+ if part.thought:+ content_str.append(f"thought: {part.thought}")+ if part.code_execution_result:+ content_str.append(f"code_execution_result: {part.code_execution_result}")+ if part.executable_code:+ content_str.append(f"executable_code: {part.executable_code}")+ if part.file_data:+ content_str.append(f"file_data: {part.file_data}")+ if part.function_call:+ content_str.append(f"function_call: {part.function_call}")+ if part.function_response:+ content_str.append(f"function_response: {part.function_response}")+ if part.inline_data:+ content_str.append(f"inline_data: {part.inline_data}")++ formatted_messages.append(f"{role}: {', '.join(content_str)}")++ prompt = "\n".join(formatted_messages) or str(args[0][0])++ input_tokens = response.usage_metadata.prompt_token_count+ output_tokens = response.usage_metadata.candidates_token_count- model = "/".join(instance._model._model_name.split("/")[3:])# Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info,- response.usage_metadata.prompt_token_count,- response.usage_metadata.candidates_token_count)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set Span attribues+ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ inference_config = kwargs.get('generation_config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use the `get` method to safely access keys in the dictionary+ value = inference_config.get(key)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage_metadata.prompt_token_count)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage_metadata.candidates_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage_metadata.total_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.candidates[0].content.parts[0].text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage_metadata.total_token_count, attributes)- metrics["genai_completion_tokens"].add(- response.usage_metadata.candidates_token_count, attributes)- metrics["genai_prompt_tokens"].add(- response.usage_metadata.prompt_token_count, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def predict_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'generate_content' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-- try:- prompt = args[0]-- model = instance._model_id- #pylint: disable=line-too-long- prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]- completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]- total_tokens = prompt_tokens + completion_tokens-- #Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes)- metrics["genai_completion_tokens"].add(- completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def predict_streaming_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'predict' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ # span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ # [str(response.candidates[0].finish_reason)])- Args:- wrapped: The original 'predict' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'predict' method.- kwargs: Keyword arguments for the 'predict' method.-- Returns:- The response from the original 'predict' method.- """-- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in wrapped(*args, **kwargs):- llmresponse += str(event)- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0]- llmresponse = llmresponse.split('TextGenerationResponse',- maxsplit=1)[0].rstrip()-- prompt_tokens = math.ceil(len(prompt) / 4)- completion_tokens = math.ceil(len(llmresponse) / 4)- total_tokens = prompt_tokens + completion_tokens-- model = instance._model_id-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)+ False)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -698,419 +407,53 @@ def predict_streaming_async(gen_ai_endpoint, version, environment, application_nspan.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,},)+ if isinstance(response.text, str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response.text is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")+span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()-- return wrapper--def start_chat_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'generate_content' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-- try:- prompt = args[0]-- model = instance._model._model_id-- #pylint: disable=line-too-long- prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]- completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]- total_tokens = prompt_tokens + completion_tokens-- #Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes)- metrics["genai_completion_tokens"].add(- completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def start_chat_streaming_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'start_chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'start_chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'start_chat' method.- kwargs: Keyword arguments for the 'start_chat' method.-- Returns:- The response from the original 'start_chat' method.- """-- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in wrapped(*args, **kwargs):- llmresponse += str(event.text)- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0]- prompt_tokens = math.ceil(len(prompt) / 4)- completion_tokens = math.ceil(len(llmresponse) / 4)- total_tokens = prompt_tokens + completion_tokens-- model = instance._model._model_id-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)+ # Return original response+ return response+except Exception as e:handle_exception(span, e)logger.error("Error in trace creation: %s", e)- return stream_generator()-- return wrapper--def embeddings_async(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'generate_content' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-- try:- prompt = args[0][0]-- model = instance._model_id-- prompt_tokens = int(response[0].statistics.token_count)-- #Calculate cost of the operation- cost = get_embed_model_cost(model,- pricing_info, prompt_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- prompt_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.pyindex b0dba24f..9ba2fc89 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/vertexai.py@@ -1,693 +1,402 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-lines"""Module for monitoring VertexAI API calls."""import logging-import math+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def generate_content(gen_ai_endpoint, version, environment, application_name, tracer,+def send_message(version, environment, application_name, tracer,pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for messages to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the VertexAI API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of VertexAI usage.trace_content: Flag indicating whether to trace the actual content.Returns:A function that wraps the chat method to add telemetry."""- def wrapper(wrapped, instance, args, kwargs):+ class TracedSyncStream:"""- Wraps the 'generate_content' API call to add telemetry.+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the response to collect message IDs and aggregated response.- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.+ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol."""+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ request_model,+ args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._input_tokens = ""+ self._output_tokens = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port+ self._request_model = request_model++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)-- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):- llmresponse += str(event.text)- prompt_tokens = event.usage_metadata.prompt_token_count- completion_tokens = event.usage_metadata.candidates_token_count- total_tokens = event.usage_metadata.total_token_count- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = str(args[0][0])-- model = "/".join(instance._model_name.split("/")[3:])-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)- return stream_generator()-- # Handling for non-streaming responses- else:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)+ self._llmresponse += str(chunk.text)+ self._input_tokens = chunk.usage_metadata.prompt_token_count+ self._output_tokens = chunk.usage_metadata.candidates_token_count+ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single string- prompt = str(args[0][0])+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages) or str(self._args[0][0])- model = "/".join(instance._model_name.split("/")[3:])# Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info,- response.usage_metadata.prompt_token_count,- response.usage_metadata.candidates_token_count)+ cost = get_chat_model_cost(self._request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)++ inference_config = self._kwargs.get('generation_config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use the `get` method to safely access keys in the dictionary+ value = inference_config.get(key)+ if value is not None:+ self._span.set_attribute(attribute, value)++ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)- span.set_attribute(SERVICE_NAME,+ self._span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage_metadata.prompt_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage_metadata.candidates_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage_metadata.total_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.candidates[0].content.parts[0].text,+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-- span.set_status(Status(StatusCode.OK))+ self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,+ request_model=self._request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage_metadata.total_token_count, attributes)- metrics["genai_completion_tokens"].add(- response.usage_metadata.candidates_token_count, attributes)- metrics["genai_prompt_tokens"].add(- response.usage_metadata.prompt_token_count, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)- # Return original response- return response-except Exception as e:- handle_exception(span, e)+ handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper---def send_message(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """+ finally:+ self._span.end()+ raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'generate_content' API call to add telemetry.+ Wraps the 'messages' API call to add telemetry.This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'generate_content' method to be wrapped.+ wrapped: The original 'messages' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.+ args: Positional arguments for the 'messages' method.+ kwargs: Keyword arguments for the 'messages' method.Returns:- The response from the original 'generate_content' method.+ The response from the original 'messages' method."""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):- llmresponse += str(event.text)- prompt_tokens = event.usage_metadata.prompt_token_count- completion_tokens = event.usage_metadata.candidates_token_count- total_tokens = event.usage_metadata.total_token_count- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0][0]-- model = "/".join(instance._model._model_name.split("/")[3:])-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ try:+ location = instance._model._location+ request_model = "/".join(instance._model._model_name.split("/")[3:])+ except:+ location = instance._location+ request_model = "/".join(instance._model_name.split("/")[3:])- span.set_status(Status(StatusCode.OK))+ server_address, server_port = location + '-aiplatform.googleapis.com', 443- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # pylint: disable=no-else-return+ if streaming:+ # Special handling for streaming response to accommodate the nature of data flow+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return stream_generator()+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port, request_model, args)# Handling for non-streaming responseselse:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()try:# Format 'messages' into a single string- prompt = args[0]+ message_prompt = kwargs.get("contents", [])+ formatted_messages = []++ for content in message_prompt:+ role = content.role+ parts = content.parts+ content_str = []++ for part in parts:+ # Collect relevant fields and handle each type of data that Part could contain+ if part.text:+ content_str.append(f"text: {part.text}")+ if part.video_metadata:+ content_str.append(f"video_metadata: {part.video_metadata}")+ if part.thought:+ content_str.append(f"thought: {part.thought}")+ if part.code_execution_result:+ content_str.append(f"code_execution_result: {part.code_execution_result}")+ if part.executable_code:+ content_str.append(f"executable_code: {part.executable_code}")+ if part.file_data:+ content_str.append(f"file_data: {part.file_data}")+ if part.function_call:+ content_str.append(f"function_call: {part.function_call}")+ if part.function_response:+ content_str.append(f"function_response: {part.function_response}")+ if part.inline_data:+ content_str.append(f"inline_data: {part.inline_data}")++ formatted_messages.append(f"{role}: {', '.join(content_str)}")++ prompt = "\n".join(formatted_messages) or str(args[0][0])++ input_tokens = response.usage_metadata.prompt_token_count+ output_tokens = response.usage_metadata.candidates_token_count- model = "/".join(instance._model._model_name.split("/")[3:])# Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info,- response.usage_metadata.prompt_token_count,- response.usage_metadata.candidates_token_count)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set Span attribues+ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ inference_config = kwargs.get('generation_config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_output_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use the `get` method to safely access keys in the dictionary+ value = inference_config.get(key)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage_metadata.prompt_token_count)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage_metadata.candidates_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage_metadata.total_token_count)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.candidates[0].content.parts[0].text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage_metadata.total_token_count, attributes)- metrics["genai_completion_tokens"].add(- response.usage_metadata.candidates_token_count, attributes)- metrics["genai_prompt_tokens"].add(- response.usage_metadata.prompt_token_count, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def predict(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'predict' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'predict' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'predict' method.- kwargs: Keyword arguments for the 'predict' method.-- Returns:- The response from the original 'predict' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-- try:- prompt = args[0]-- model = instance._model_id- #pylint: disable=line-too-long- prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]- completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]- total_tokens = prompt_tokens + completion_tokens-- #Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes)- metrics["genai_completion_tokens"].add(- completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def predict_streaming(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'predict' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ # span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ # [str(response.candidates[0].finish_reason)])- Args:- wrapped: The original 'predict' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'predict' method.- kwargs: Keyword arguments for the 'predict' method.-- Returns:- The response from the original 'predict' method.- """-- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):- llmresponse += str(event)- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0]- llmresponse = llmresponse.split('TextGenerationResponse',- maxsplit=1)[0].rstrip()-- prompt_tokens = math.ceil(len(prompt) / 4)- completion_tokens = math.ceil(len(llmresponse) / 4)- total_tokens = prompt_tokens + completion_tokens-- model = instance._model_id-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)+ False)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -698,419 +407,53 @@ def predict_streaming(gen_ai_endpoint, version, environment, application_name, tspan.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,},)+ if isinstance(response.text, str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response.text is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")+span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()-- return wrapper--def start_chat(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'start_chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'start_chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'start_chat' method.- kwargs: Keyword arguments for the 'start_chat' method.-- Returns:- The response from the original 'start_chat' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-- try:- prompt = args[0]-- model = instance._model._model_id-- #pylint: disable=line-too-long- prompt_tokens = response._prediction_response.metadata["tokenMetadata"]["inputTokenCount"]["totalTokens"]- completion_tokens = response._prediction_response.metadata["tokenMetadata"]["outputTokenCount"]["totalTokens"]- total_tokens = prompt_tokens + completion_tokens-- #Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes)- metrics["genai_completion_tokens"].add(- completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def start_chat_streaming(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'start_chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'start_chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'start_chat' method.- kwargs: Keyword arguments for the 'start_chat' method.-- Returns:- The response from the original 'start_chat' method.- """-- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):- llmresponse += str(event.text)- yield event-- # Handling exception ensure observability without disrupting operation- try:- prompt = args[0]- prompt_tokens = math.ceil(len(prompt) / 4)- completion_tokens = math.ceil(len(llmresponse) / 4)- total_tokens = prompt_tokens + completion_tokens-- model = instance._model._model_id-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)+ # Return original response+ return response+except Exception as e:handle_exception(span, e)logger.error("Error in trace creation: %s", e)- return stream_generator()-- return wrapper--def embeddings(gen_ai_endpoint, version, environment, application_name, tracer,- pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for messages to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'generate_content' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'generate_content' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate_content' method.- kwargs: Keyword arguments for the 'generate_content' method.-- Returns:- The response from the original 'generate_content' method.- """-- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-- try:- prompt = args[0][0]-- model = instance._model_id-- prompt_tokens = int(response[0].statistics.token_count)-- #Calculate cost of the operation- cost = get_embed_model_cost(model,- pricing_info, prompt_tokens)-- # Set Span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VERTEXAI,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- prompt_tokens, attributes)- metrics["genai_prompt_tokens"].add(- prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/vllm/__init__.py b/sdk/python/src/openlit/instrumentation/vllm/__init__.pyindex f13468b8..337a79a6 100644--- a/sdk/python/src/openlit/instrumentation/vllm/__init__.py+++ b/sdk/python/src/openlit/instrumentation/vllm/__init__.py@@ -34,7 +34,7 @@ class VLLMInstrumentor(BaseInstrumentor):wrap_function_wrapper("vllm","LLM.generate",- generate("vllm.generate", version, environment, application_name,+ generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/vllm/vllm.py b/sdk/python/src/openlit/instrumentation/vllm/vllm.pyindex a7e4aedd..7e6facc7 100644--- a/sdk/python/src/openlit/instrumentation/vllm/vllm.py+++ b/sdk/python/src/openlit/instrumentation/vllm/vllm.py@@ -1,24 +1,29 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring vLLM API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT-from openlit.__helpers import handle_exception, general_tokens+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ general_tokens,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def generate(gen_ai_endpoint, version, environment, application_name,+def generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for generate to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the vLLM API.@@ -47,28 +52,47 @@ def generate(gen_ai_endpoint, version, environment, application_name,The response from the original 'generate' method."""+ server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)+ request_model = instance.llm_engine.model_config.model or "facebook/opt-125m"++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"+# pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()try:- model = instance.llm_engine.model_config.model or "facebook/opt-125m"# Set base span attribuesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,SemanticConvetion.GEN_AI_SYSTEM_VLLM)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")++ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)+ environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+input_tokens = 0output_tokens = 0cost = 0@@ -95,37 +119,43 @@ def generate(gen_ai_endpoint, version, environment, application_name,attributes=completion_attributes,)- total_tokens = input_tokens + output_tokens-span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)+ input_tokens + output_tokens)++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model, pricing_info,+ input_tokens, output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_VLLM,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_VLLM,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)metrics["genai_completion_tokens"].add(output_tokens, attributes)metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)