Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update AssemblyAI, Bedrock, Gemini, ElevenLabs and Azure AI Inference Instrumentation for compatibility with OTel Semconv (#620)
Patcher2f5de0a279ba
diff --git a/sdk/python/.pylintrc b/sdk/python/.pylintrcindex 9542de74..fe2409c9 100644--- a/sdk/python/.pylintrc+++ b/sdk/python/.pylintrc@@ -24,4 +24,6 @@ disable=unused-argument,too-many-branches,too-many-locals,- too-many-lines\ No newline at end of file+ too-many-lines,+ no-name-in-module,+ protected-access,\ No newline at end of filediff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 9e20da8b..dde0383f 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.8"+version = "1.33.9"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 7ae739b5..01a12c6c 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -257,9 +257,15 @@ def set_server_address_and_port(client_instance: Any,using defaults if none found or values are None."""+ # Try getting base_url from multiple potential attributesbase_client = getattr(client_instance, "_client", None)base_url = getattr(base_client, "base_url", None)+ if not base_url:+ # Attempt to get endpoint from instance._config.endpoint if base_url is not set+ config = getattr(client_instance, "_config", None)+ base_url = getattr(config, "endpoint", None)+if base_url:if isinstance(base_url, str):url = urlparse(base_url)@@ -269,7 +275,7 @@ def set_server_address_and_port(client_instance: Any,server_address = getattr(base_url, "host", None) or default_server_addressport_attr = getattr(base_url, "port", None)server_port = port_attr if port_attr is not None else default_server_port- else: # no base_url provided; use defaults.+ else: # no base_url or endpoint provided; use defaults.server_address = default_server_addressserver_port = default_server_portdiff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex f14fdb6c..8bba065f 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -251,7 +251,7 @@ def init("gpt4all": "gpt4all","elevenlabs": "elevenlabs","vllm": "vllm",- "google-ai-studio": "google.generativeai",+ "google-ai-studio": "google.genai","azure-ai-inference": "azure.ai.inference","langchain": "langchain","llama_index": "llama_index",diff --git a/sdk/python/src/openlit/instrumentation/assemblyai/__init__.py b/sdk/python/src/openlit/instrumentation/assemblyai/__init__.pyindex 897c3e60..2280414f 100644--- a/sdk/python/src/openlit/instrumentation/assemblyai/__init__.py+++ b/sdk/python/src/openlit/instrumentation/assemblyai/__init__.py@@ -34,7 +34,7 @@ class AssemblyAIInstrumentor(BaseInstrumentor):wrap_function_wrapper("assemblyai.transcriber","Transcriber.transcribe",- transcribe("assemblyai.transcribe", version, environment, application_name,+ transcribe(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/assemblyai/assemblyai.py b/sdk/python/src/openlit/instrumentation/assemblyai/assemblyai.pyindex 2bdfdf96..62db15c0 100644--- a/sdk/python/src/openlit/instrumentation/assemblyai/assemblyai.py+++ b/sdk/python/src/openlit/instrumentation/assemblyai/assemblyai.py@@ -1,19 +1,23 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Assembly AI API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_audio_model_cost-from openlit.__helpers import handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_audio_model_cost,+ handle_exception,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def transcribe(gen_ai_endpoint, version, environment, application_name,+def transcribe(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for creating speech audio to collect metrics.@@ -48,43 +52,56 @@ def transcribe(gen_ai_endpoint, version, environment, application_name,The response from the original 'transcribe' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, 'api.assemblyai.com', 443)+ request_model = kwargs.get('speech_model', 'best')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO} {request_model}'++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()try:- llm_model = response.speech_model if response.speech_model else "best"-# Calculate cost of the operation- cost = get_audio_model_cost(llm_model,+ cost = get_audio_model_cost(request_model,pricing_info, None, response.audio_duration)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI)+ # Set Span attributes (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')++ # Set Span attributes (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- llm_model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_DURATION,response.audio_duration)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: response.audio_url,- },+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: response.audio_url,+ },)span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,@@ -96,30 +113,29 @@ def transcribe(gen_ai_endpoint, version, environment, application_name,span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- llm_model,- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,+ system=SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.pyindex 81fcf234..cb499bed 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py@@ -38,7 +38,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):wrap_function_wrapper("azure.ai.inference","ChatCompletionsClient.complete",- complete("azure_ai.complete", version, environment, application_name,+ complete(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -46,7 +46,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):wrap_function_wrapper("azure.ai.inference","EmbeddingsClient.embed",- embedding("azure_ai.embed", version, environment, application_name,+ embedding(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -54,7 +54,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):wrap_function_wrapper("azure.ai.inference.aio","ChatCompletionsClient.complete",- async_complete("azure_ai.complete", version, environment, application_name,+ async_complete(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -62,7 +62,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):wrap_function_wrapper("azure.ai.inference.aio","EmbeddingsClient.embed",- async_embedding("azure_ai.embed", version, environment, application_name,+ async_embedding(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.pyindex e22b638d..b19578da 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py@@ -1,15 +1,20 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment, protected-access"""Module for monitoring Azure AI Inference API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (- handle_exception,get_chat_model_cost,get_embed_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port,general_tokens)from openlit.semcov import SemanticConvetion@@ -17,13 +22,12 @@ from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_complete(gen_ai_endpoint, version, environment, application_name,+def async_complete(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Azure AI Inference API.@@ -35,165 +39,275 @@ def async_complete(gen_ai_endpoint, version, environment, application_name,A function that wraps the chat method to add telemetry."""+ class TracedAsyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'az.ai.inference.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._system_fingerprint = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ # Collect message IDs and aggregated response from events+ if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and+ 'content' in chunked.get('choices')[0].get('delta'))):++ content = chunked.get('choices')[0].get('delta').get('content')+ if content:+ self._llmresponse += content+ self._response_id = chunked.get('id')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')+ self._system_fingerprint = chunked.get('system_fingerprint')+ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate tokens using input prompt and aggregated response+ input_tokens = general_tokens(prompt)+ output_tokens = general_tokens(self._llmresponse)++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+async def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'chat' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'chat' method.+ The response from the original 'chat.completions' method."""- # pylint: disable=no-else-return- if kwargs.get("stream", False) is True:- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- if chunk.choices:- # Collect message IDs and aggregated response from events- content = chunk.choices[0].delta.content- if content:- llmresponse += content-- yield chunk- response_id = chunk.id-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- model = kwargs.get("model", "phi3-mini-4k")-- # Calculate tokens using input prompt and aggregated response- input_tokens = general_tokens(prompt)- output_tokens = general_tokens(llmresponse)-- total_tokens = input_tokens + output_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ # Check if streaming is enabled for the API call+ streaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "gpt-4o")- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ # pylint: disable=no-else-return+ if streaming:+ # Special handling for streaming response to accommodate the nature of data flow+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)- return stream_generator()+ # Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string@@ -205,7 +319,6 @@ def async_complete(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -215,43 +328,66 @@ def async_complete(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- model = kwargs.get("model", "phi3-mini-4k")+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set base span attribues+ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0]["finish_reason"]])+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ response_dict.get('system_fingerprint'))+ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -259,49 +395,53 @@ def async_complete(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content,- },- )- input_tokens = response.usage.prompt_tokens- output_tokens = response.usage.completion_tokens- total_tokens = response.usage.total_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens, output_tokens)+ for i in range(kwargs.get('n',1)):+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)metrics["genai_completion_tokens"].add(output_tokens, attributes)metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)@@ -318,18 +458,17 @@ def async_complete(gen_ai_endpoint, version, environment, application_name,return wrapper-def async_embedding(gen_ai_endpoint, version, environment, application_name,+def async_embedding(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure AI Inference API.+ application_name: Name of the application using the Azure Inference API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure AI Inference usage.+ pricing_info: Information used for calculating the cost of Azure Inference usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -353,69 +492,84 @@ def async_embedding(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "text-embedding-ada-002")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()+ response_dict = response_as_dict(response)try:+ input_tokens = response_dict.get('usage').get('prompt_tokens')+# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get("model", "text-embedding-ada-002"),- pricing_info, response.usage.prompt_tokens)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)- # Set Span attributes+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "text-embedding-ada-002"))+ request_model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get("encoding_format", "float"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,- kwargs.get("dimensions", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))+ [kwargs.get('encoding_format', 'float')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)+ input_tokens)++ # Set Span attributes (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("input", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("input", "")),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "text-embedding-ada-002")- }-+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.pyindex 2612afc8..f5c56d1f 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py@@ -1,15 +1,20 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment, protected-access"""Module for monitoring Azure AI Inference API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (- handle_exception,get_chat_model_cost,get_embed_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port,general_tokens)from openlit.semcov import SemanticConvetion@@ -17,13 +22,12 @@ from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def complete(gen_ai_endpoint, version, environment, application_name,+def complete(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Azure AI Inference API.@@ -35,165 +39,275 @@ def complete(gen_ai_endpoint, version, environment, application_name,A function that wraps the chat method to add telemetry."""+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'az.ai.inference.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._system_fingerprint = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ # Collect message IDs and aggregated response from events+ if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and+ 'content' in chunked.get('choices')[0].get('delta'))):++ content = chunked.get('choices')[0].get('delta').get('content')+ if content:+ self._llmresponse += content+ self._response_id = chunked.get('id')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')+ self._system_fingerprint = chunked.get('system_fingerprint')+ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate tokens using input prompt and aggregated response+ input_tokens = general_tokens(prompt)+ output_tokens = general_tokens(self._llmresponse)++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'chat' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'chat' method.+ The response from the original 'chat.completions' method."""- # pylint: disable=no-else-return- if kwargs.get("stream", False) is True:- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- if chunk.choices:- # Collect message IDs and aggregated response from events- content = chunk.choices[0].delta.content- if content:- llmresponse += content-- yield chunk- response_id = chunk.id-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- model = kwargs.get("model", "phi3-mini-4k")-- # Calculate tokens using input prompt and aggregated response- input_tokens = general_tokens(prompt)- output_tokens = general_tokens(llmresponse)-- total_tokens = input_tokens + output_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ # Check if streaming is enabled for the API call+ streaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "gpt-4o")- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ # pylint: disable=no-else-return+ if streaming:+ # Special handling for streaming response to accommodate the nature of data flow+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)- return stream_generator()+ # Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string@@ -205,7 +319,6 @@ def complete(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -215,43 +328,66 @@ def complete(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- model = kwargs.get("model", "phi3-mini-4k")+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set base span attribues+ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0]["finish_reason"]])+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ response_dict.get('system_fingerprint'))+ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -259,49 +395,53 @@ def complete(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content,- },- )- input_tokens = response.usage.prompt_tokens- output_tokens = response.usage.completion_tokens- total_tokens = response.usage.total_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens, output_tokens)+ for i in range(kwargs.get('n',1)):+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)metrics["genai_completion_tokens"].add(output_tokens, attributes)metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)@@ -318,18 +458,17 @@ def complete(gen_ai_endpoint, version, environment, application_name,return wrapper-def embedding(gen_ai_endpoint, version, environment, application_name,+def embedding(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure AI Inference API.+ application_name: Name of the application using the Azure Inference API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure AI Inference usage.+ pricing_info: Information used for calculating the cost of Azure Inference usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -353,69 +492,84 @@ def embedding(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "text-embedding-ada-002")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()+ response_dict = response_as_dict(response)try:+ input_tokens = response_dict.get('usage').get('prompt_tokens')+# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get("model", "text-embedding-ada-002"),- pricing_info, response.usage.prompt_tokens)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)- # Set Span attributes+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "text-embedding-ada-002"))+ request_model)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get("encoding_format", "float"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,- kwargs.get("dimensions", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))+ [kwargs.get('encoding_format', 'float')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)+ input_tokens)++ # Set Span attributes (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("input", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("input", "")),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "text-embedding-ada-002")- }-+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/bedrock/__init__.py b/sdk/python/src/openlit/instrumentation/bedrock/__init__.pyindex f26c4e86..71a1f1b8 100644--- a/sdk/python/src/openlit/instrumentation/bedrock/__init__.py+++ b/sdk/python/src/openlit/instrumentation/bedrock/__init__.py@@ -32,7 +32,7 @@ class BedrockInstrumentor(BaseInstrumentor):wrap_function_wrapper("botocore.client","ClientCreator.create_client",- converse("bedrock.converse", version, environment, application_name,+ converse(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.pyindex 281dd991..8cda881d 100644--- a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py+++ b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py@@ -1,17 +1,22 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-branches"""Module for monitoring Amazon Bedrock API calls."""import logging+import timefrom botocore.response import StreamingBodyfrom botocore.exceptions import ReadTimeoutError, ResponseStreamingErrorfrom urllib3.exceptions import ProtocolError as URLLib3ProtocolErrorfrom urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutErrorfrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_chat_model_cost-from openlit.__helpers import handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ response_as_dict,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operations@@ -47,8 +52,7 @@ class CustomStreamWrapper(StreamingBody):return data_chunk--def converse(gen_ai_endpoint, version, environment, application_name, tracer,+def converse(version, environment, application_name, tracer,pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for messages to collect metrics.@@ -91,52 +95,104 @@ def converse(gen_ai_endpoint, version, environment, application_name, tracer,Returns:The modified response with telemetry."""- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:++ server_address, server_port = set_server_address_and_port(instance, 'aws.amazon.com', 443)+ request_model = method_kwargs.get('modelId', 'amazon.titan-text-express-v1')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = original_method(*method_args, **method_kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:- message_prompt = method_kwargs.get("messages", "")+ message_prompt = method_kwargs.get('messages', '')formatted_messages = []for message in message_prompt:- role = message["role"]- content = message["content"]+ role = message['role']+ content = message['content']if isinstance(content, list):- # pylint: disable=line-too-longcontent_str = ", ".join(f'text: {item["text"]}' for item in content if "text" in item)- formatted_messages.append(f"{role}: {content_str}")+ formatted_messages.append(f'{role}: {content_str}')else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)+ formatted_messages.append(f'{role}: {content}')+ prompt = '\n'.join(formatted_messages)++ input_tokens = response_dict.get('usage').get('inputTokens')+ output_tokens = response_dict.get('usage').get('outputTokens')- model = method_kwargs.get("modelId", "amazon.titan-text-express-v1")- input_tokens = response["usage"]["inputTokens"]- output_tokens = response["usage"]["outputTokens"]+ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model, pricing_info,+ input_tokens, output_tokens)++ llm_response = response_dict.get('output').get('message').get('content')[0].get('text')- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')+ span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,SemanticConvetion.GEN_AI_SYSTEM_AWS_BEDROCK)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ inference_config = method_kwargs.get('inferenceConfig', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequencyPenalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'maxTokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presencePenalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stopSequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'topP'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'topK'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ value = inference_config.get(key)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get('ResponseMetadata').get('RequestId'))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ if isinstance(llm_response, str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')+ else:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,input_tokens + output_tokens)-- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens,- output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -148,42 +204,43 @@ def converse(gen_ai_endpoint, version, environment, application_name, tracer,span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response["output"]["message"]["content"][0]["text"],+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AWS_BEDROCK,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AWS_BEDROCK,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics['genai_client_usage_tokens'].record(input_tokens + output_tokens, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_server_ttft'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_completion_tokens'].add(output_tokens, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)return responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn response@@ -192,7 +249,7 @@ def converse(gen_ai_endpoint, version, environment, application_name, tracer,client = wrapped(*args, **kwargs)# Replace the original method with the instrumented one- if kwargs.get("service_name") == "bedrock-runtime":+ if kwargs.get('service_name') == 'bedrock-runtime':original_invoke_model = client.converseclient.converse = lambda *args, **kwargs: converse_wrapper(original_invoke_model,*args, **kwargs)diff --git a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.pyindex ac9b78d7..2aa83a09 100644--- a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py@@ -244,7 +244,7 @@ def async_chat(version, environment, application_name, tracer,kwargs.get("stop_sequences", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,kwargs.get("temperature", 0.3))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,kwargs.get("k", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,kwargs.get("p", 1.0))diff --git a/sdk/python/src/openlit/instrumentation/cohere/cohere.py b/sdk/python/src/openlit/instrumentation/cohere/cohere.pyindex 7158c604..4bd4e132 100644--- a/sdk/python/src/openlit/instrumentation/cohere/cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/cohere.py@@ -244,7 +244,7 @@ def chat(version, environment, application_name, tracer,kwargs.get("stop_sequences", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,kwargs.get("temperature", 0.3))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,kwargs.get("k", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,kwargs.get("p", 1.0))diff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.pyindex 529a6817..54440aec 100644--- a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py@@ -1,13 +1,17 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Ollama API calls."""import logging+import time+from urllib.parse import urlparsefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_audio_model_cost-from openlit.__helpers import handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_audio_model_cost,+ handle_exception,+ create_metrics_attributes,+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operations@@ -19,7 +23,6 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,Generates a telemetry wrapper for creating speech audio to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the ElevenLabs API.@@ -47,73 +50,96 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,Returns:The response from the original 'generate' method."""+ url = urlparse(instance._client_wrapper._base_url)+ server_address, server_port = url.hostname, url.port or 443+ request_model = kwargs.get('model', kwargs.get('model_id', 'eleven_multilingual_v2'))- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO} {request_model}'+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()+ response = await wrapped(*args, **kwargs)+ end_time = time.time()+ for key, value in instance.__dict__.items():+ print(f'{key}: {value}')try:# Calculate cost of the operation- cost = get_audio_model_cost(kwargs.get("model", "eleven_multilingual_v2"),- pricing_info, kwargs.get("text", ""))+ cost = get_audio_model_cost(request_model,+ pricing_info, kwargs.get('text', ''))# Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "eleven_multilingual_v2"))- if isinstance(kwargs.get("voice", "Rachel"), str):+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'audio')++ # Set Span attributes (Extras)+ if gen_ai_endpoint == 'elevenlabs.generate':+ if isinstance(kwargs.get('voice', 'Rachel'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,+ kwargs.get('voice', 'Rachel'))+ else:span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,- kwargs.get("voice", "Rachel"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,- kwargs.get("output_format", "mp3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SETTINGS,- str(kwargs.get("voice_settings", "")))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ kwargs.get('voice_id', ''))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,+ kwargs.get('output_format', 'mp3'))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SETTINGS,+ str(kwargs.get('voice_settings', '')))+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("text", "")),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('text', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "eleven_multilingual_v2")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,+ system=SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.pyindex b4df23c0..7b3d9cec 100644--- a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py@@ -1,13 +1,17 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Ollama API calls."""import logging+import time+from urllib.parse import urlparsefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_audio_model_cost-from openlit.__helpers import handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_audio_model_cost,+ handle_exception,+ create_metrics_attributes,+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operations@@ -19,7 +23,6 @@ def generate(gen_ai_endpoint, version, environment, application_name,Generates a telemetry wrapper for creating speech audio to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the ElevenLabs API.@@ -48,78 +51,96 @@ def generate(gen_ai_endpoint, version, environment, application_name,The response from the original 'generate' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)+ url = urlparse(instance._client_wrapper._base_url)+ server_address, server_port = url.hostname, url.port or 443+ request_model = kwargs.get('model', kwargs.get('model_id', 'eleven_multilingual_v2'))++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO} {request_model}'+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)+ end_time = time.time()+ for key, value in instance.__dict__.items():+ print(f'{key}: {value}')try:# Calculate cost of the operation- cost = get_audio_model_cost(kwargs.get("model", "eleven_multilingual_v2"),- pricing_info, kwargs.get("text", ""))+ cost = get_audio_model_cost(request_model,+ pricing_info, kwargs.get('text', ''))# Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- if gen_ai_endpoint == "elevenlabs.generate":- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "eleven_multilingual_v2"))- if isinstance(kwargs.get("voice", "Rachel"), str):+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ASSEMBLYAI)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'audio')++ # Set Span attributes (Extras)+ if gen_ai_endpoint == 'elevenlabs.generate':+ if isinstance(kwargs.get('voice', 'Rachel'), str):span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,- kwargs.get("voice", "Rachel"))+ kwargs.get('voice', 'Rachel'))else:- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model_id", "eleven_multilingual_v2"))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,- kwargs.get("voice_id", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,- kwargs.get("output_format", "mp3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SETTINGS,- str(kwargs.get("voice_settings", "")))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ kwargs.get('voice_id', ''))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,+ kwargs.get('output_format', 'mp3'))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SETTINGS,+ str(kwargs.get('voice_settings', '')))+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("text", "")),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('text', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "eleven_multilingual_v2")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO,+ system=SemanticConvetion.GEN_AI_SYSTEM_ELEVENLABS,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/__init__.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/__init__.pyindex 897f8e9a..2219b009 100644--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/__init__.py+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/__init__.py@@ -14,11 +14,11 @@ from openlit.instrumentation.google_ai_studio.async_google_ai_studio import (async_generate)-_instruments = ("google-generativeai >= 0.2.0",)+_instruments = ("google-genai >= 1.3.0",)class GoogleAIStudioInstrumentor(BaseInstrumentor):"""- An instrumentor for google-generativeai's client library.+ An instrumentor for google-genai's client library."""def instrumentation_dependencies(self) -> Collection[str]:@@ -32,21 +32,21 @@ class GoogleAIStudioInstrumentor(BaseInstrumentor):pricing_info = kwargs.get("pricing_info", {})trace_content = kwargs.get("trace_content", False)disable_metrics = kwargs.get("disable_metrics")- version = importlib.metadata.version("google-generativeai")+ version = importlib.metadata.version("google-genai")# sync generatewrap_function_wrapper(- "google.generativeai.generative_models",- "GenerativeModel.generate_content",- generate("google_ai_studio.generate_content", version, environment, application_name,+ "google.genai.models",+ "Models.generate_content",+ generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)# async generatewrap_function_wrapper(- "google.generativeai.generative_models",- "GenerativeModel.generate_content_async",- async_generate("google_ai_studio.generate_content", version, environment,+ "google.genai.models",+ "AsyncModels.generate_content",+ async_generate(version, environment,application_name, tracer, pricing_info, trace_content, metrics,disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.pyindex 943453ad..bb4721d4 100644--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py@@ -1,21 +1,24 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment, protected-access"""Module for monitoring Google AI Studio API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (- handle_exception,get_chat_model_cost,+ handle_exception,+ response_as_dict,+ create_metrics_attributes,+ set_server_address_and_port)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_generate(gen_ai_endpoint, version, environment, application_name,+def async_generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.@@ -35,229 +38,190 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,async def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'chat' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'chat' method.+ The response from the original 'chat.completions' method."""- # pylint: disable=no-else-return- if kwargs.get('stream', False) is True:- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- # Collect message IDs and aggregated response from events- content = chunk.text- if content:- llmresponse += content-- input_tokens = chunk.usage_metadata.prompt_token_count- output_tokens = chunk.usage_metadata.candidates_token_count- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- prompt = ""- for arg in args:- if isinstance(arg, str):- prompt = f"{prompt}{arg}\n"- elif isinstance(arg, list):- for subarg in arg:- prompt = f"{prompt}{subarg}\n"- if hasattr(instance, "_model_id"):- model = instance._model_id- if hasattr(instance, "_model_name"):- model = instance._model_name.replace("publishers/google/models/", "")- if model.startswith("models/"):- model = model[len("models/"):]-- total_tokens = input_tokens + output_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens,- output_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,++ server_address, server_port = set_server_address_and_port(instance, "generativelanguage.googleapis.com", 443)+ request_model = kwargs.get("model", "gemini-2.0-flash")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = await wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)++ try:+ # Format 'messages' into a single string+ message_prompt = kwargs.get("contents", [])+ formatted_messages = []++ for content in message_prompt:+ role = content.role+ parts = content.parts+ content_str = []++ for part in parts:+ # Collect relevant fields and handle each type of data that Part could contain+ if part.text:+ content_str.append(f"text: {part.text}")+ if part.video_metadata:+ content_str.append(f"video_metadata: {part.video_metadata}")+ if part.thought:+ content_str.append(f"thought: {part.thought}")+ if part.code_execution_result:+ content_str.append(f"code_execution_result: {part.code_execution_result}")+ if part.executable_code:+ content_str.append(f"executable_code: {part.executable_code}")+ if part.file_data:+ content_str.append(f"file_data: {part.file_data}")+ if part.function_call:+ content_str.append(f"function_call: {part.function_call}")+ if part.function_response:+ content_str.append(f"function_response: {part.function_response}")+ if part.inline_data:+ content_str.append(f"inline_data: {part.inline_data}")++ formatted_messages.append(f"{role}: {', '.join(content_str)}")++ prompt = "\n".join(formatted_messages)++ input_tokens = response_dict.get('usage_metadata').get('prompt_token_count')+ output_tokens = response_dict.get('usage_metadata').get('candidates_token_count')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-- try:- prompt = ""- for arg in args:- if isinstance(arg, str):- prompt = f"{prompt}{arg}\n"- elif isinstance(arg, list):- for subarg in arg:- prompt = f"{prompt}{subarg}\n"- if hasattr(instance, "_model_id"):- model = instance._model_id- if hasattr(instance, "_model_name"):- model = instance._model_name.replace("publishers/google/models/", "")- if model.startswith("models/"):- model = model[len("models/"):]-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)-- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- prompt_tokens = response.usage_metadata.prompt_token_count- completion_tokens = response.usage_metadata.candidates_token_count- total_tokens = response.usage_metadata.total_token_count- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response++ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GEMINI)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ inference_config = kwargs.get('config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use getattr to get the attribute value from the object+ value = getattr(inference_config, key, None)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model_version'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [str(response_dict.get('candidates')[0].get('finish_reason'))])++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,+ },+ )++ if isinstance(response_dict.get('text'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('text') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GEMINI,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model_version'),+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/google_ai_studio.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/google_ai_studio.pyindex 3c2eb8ec..7a54b4cf 100644--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/google_ai_studio.py+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/google_ai_studio.py@@ -1,21 +1,24 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment, protected-access"""Module for monitoring Google AI Studio API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (- handle_exception,get_chat_model_cost,+ handle_exception,+ response_as_dict,+ create_metrics_attributes,+ set_server_address_and_port)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def generate(gen_ai_endpoint, version, environment, application_name,+def generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.@@ -35,230 +38,190 @@ def generate(gen_ai_endpoint, version, environment, application_name,def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'chat' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'chat' method.+ The response from the original 'chat.completions' method."""- # pylint: disable=no-else-return- if kwargs.get("stream", False) is True:- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- # Collect message IDs and aggregated response from events- content = chunk.text- if content:- llmresponse += content-- input_tokens = chunk.usage_metadata.prompt_token_count- output_tokens = chunk.usage_metadata.candidates_token_count- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- prompt = ""- for arg in args:- if isinstance(arg, str):- prompt = f"{prompt}{arg}\n"- elif isinstance(arg, list):- for subarg in arg:- prompt = f"{prompt}{subarg}\n"- if hasattr(instance, "_model_id"):- model = instance._model_id- if hasattr(instance, "_model_name"):- model = instance._model_name.replace("publishers/google/models/", "")- if model.startswith("models/"):- model = model[len("models/"):]-- total_tokens = input_tokens + output_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, input_tokens,- output_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,++ server_address, server_port = set_server_address_and_port(instance, "generativelanguage.googleapis.com", 443)+ request_model = kwargs.get("model", "gemini-2.0-flash")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)++ try:+ # Format 'messages' into a single string+ message_prompt = kwargs.get("contents", [])+ formatted_messages = []++ for content in message_prompt:+ role = content.role+ parts = content.parts+ content_str = []++ for part in parts:+ # Collect relevant fields and handle each type of data that Part could contain+ if part.text:+ content_str.append(f"text: {part.text}")+ if part.video_metadata:+ content_str.append(f"video_metadata: {part.video_metadata}")+ if part.thought:+ content_str.append(f"thought: {part.thought}")+ if part.code_execution_result:+ content_str.append(f"code_execution_result: {part.code_execution_result}")+ if part.executable_code:+ content_str.append(f"executable_code: {part.executable_code}")+ if part.file_data:+ content_str.append(f"file_data: {part.file_data}")+ if part.function_call:+ content_str.append(f"function_call: {part.function_call}")+ if part.function_response:+ content_str.append(f"function_response: {part.function_response}")+ if part.inline_data:+ content_str.append(f"inline_data: {part.inline_data}")++ formatted_messages.append(f"{role}: {', '.join(content_str)}")++ prompt = "\n".join(formatted_messages)++ input_tokens = response_dict.get('usage_metadata').get('prompt_token_count')+ output_tokens = response_dict.get('usage_metadata').get('candidates_token_count')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-- try:- prompt = ""- for arg in args:- if isinstance(arg, str):- prompt = f"{prompt}{arg}\n"- elif isinstance(arg, list):- for subarg in arg:- prompt = f"{prompt}{subarg}\n"-- if hasattr(instance, "_model_id"):- model = instance._model_id- if hasattr(instance, "_model_name"):- model = instance._model_name.replace("publishers/google/models/", "")- if model.startswith("models/"):- model = model[len("models/"):]-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)-- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,- },- )-- prompt_tokens = response.usage_metadata.prompt_token_count- completion_tokens = response.usage_metadata.candidates_token_count- total_tokens = response.usage_metadata.total_token_count- # Calculate cost of the operation- cost = get_chat_model_cost(model,- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GOOGLE_AI_STUDIO,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response++ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GEMINI)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ inference_config = kwargs.get('config', {})++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'frequency_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, 'presence_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop_sequences'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use getattr to get the attribute value from the object+ value = getattr(inference_config, key, None)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model_version'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [str(response_dict.get('candidates')[0].get('finish_reason'))])++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.text,+ },+ )++ if isinstance(response_dict.get('text'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('text') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GEMINI,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model_version'),+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.pyindex 19739159..5d1acde7 100644--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py@@ -580,7 +580,7 @@ def async_embedding(version, environment, application_name,request_model=request_model,server_address=server_address,server_port=server_port,- response_model=response_dict.get('model'),+ response_model=request_model,)metrics["genai_client_usage_tokens"].record(input_tokens, attributesdiff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.pyindex 1ec61b2f..7c39adb2 100644--- a/sdk/python/src/openlit/instrumentation/openai/openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py@@ -580,7 +580,7 @@ def embedding(version, environment, application_name,request_model=request_model,server_address=server_address,server_port=server_port,- response_model=response_dict.get('model'),+ response_model=request_model,)metrics["genai_client_usage_tokens"].record(input_tokens, attributesdiff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 4e09da3c..616722a5 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -98,7 +98,7 @@ class SemanticConvetion:GEN_AI_SYSTEM_GPT4ALL = "gpt4all"GEN_AI_SYSTEM_ELEVENLABS = "elevenlabs"GEN_AI_SYSTEM_VLLM = "vLLM"- GEN_AI_SYSTEM_GOOGLE_AI_STUDIO = "google-ai-studio"+ GEN_AI_SYSTEM_GOOGLE_AI_STUDIO = "google.ai.studio"GEN_AI_SYSTEM_REKAAI = "rekaai"GEN_AI_SYSTEM_PREMAI = "premai"GEN_AI_SYSTEM_LANGCHAIN = "langchain"