Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Use Events API In Azure AI Inference OpenTelemetry Instrumentation (#662)
Patcher117fe1132760
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex a4ff4b30..e9ddd19d 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.17"+version = "1.33.18"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.pyindex d946b80d..b4ea879c 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py@@ -4,13 +4,11 @@ from typing import Collectionimport importlib.metadatafrom opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapper-from openlit.instrumentation.azure_ai_inference.azure_ai_inference import (- complete, embedding+ complete)-from openlit.instrumentation.azure_ai_inference.async_azure_ai_inference import (- async_complete, async_embedding+ async_complete)_instruments = ('azure-ai-inference >= 1.0.0b4',)@@ -27,6 +25,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):application_name = kwargs.get('application_name', 'default')environment = kwargs.get('environment', 'default')tracer = kwargs.get('tracer')+ event_provider = kwargs.get('event_provider')metrics = kwargs.get('metrics_dict')pricing_info = kwargs.get('pricing_info', {})capture_message_content = kwargs.get('capture_message_content', False)@@ -38,15 +37,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):'azure.ai.inference','ChatCompletionsClient.complete',complete(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),- )-- # sync embedding- wrap_function_wrapper(- 'azure.ai.inference',- 'EmbeddingsClient.embed',- embedding(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),)# async generate@@ -54,15 +45,7 @@ class AzureAIInferenceInstrumentor(BaseInstrumentor):'azure.ai.inference.aio','ChatCompletionsClient.complete',async_complete(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),- )-- # async embedding- wrap_function_wrapper(- 'azure.ai.inference.aio',- 'EmbeddingsClient.embed',- async_embedding(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),)def _uninstrument(self, **kwargs):diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.pyindex 429473ed..522595f1 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py@@ -4,18 +4,15 @@ Module for monitoring Azure AI Inference API calls.import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,- get_embed_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,set_server_address_and_port,- general_tokens+)+from openlit.instrumentation.azure_ai_inference.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,)from openlit.semcov import SemanticConvetion@@ -23,37 +20,21 @@ from openlit.semcov import SemanticConvetionlogger = logging.getLogger(__name__)def async_complete(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure AI Inference API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure AI Inference usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for GenAI function call"""class TracedAsyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -61,12 +42,13 @@ def async_complete(version, environment, application_name,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""self._response_id = ""self._response_model = ""self._finish_reason = ""- self._system_fingerprint = ""+ self._input_tokens = 0+ self._output_tokens = 0self._args = argsself._kwargs = kwargs@@ -95,197 +77,33 @@ def async_complete(version, environment, application_name,async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from events- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and- 'content' in chunked.get('choices')[0].get('delta'))):-- content = chunked.get('choices')[0].get('delta').get('content')- if content:- self._llmresponse += content- self._response_id = chunked.get('id')- self._response_model = chunked.get('model')- self._finish_reason = chunked.get('choices')[0].get('finish_reason')- self._system_fingerprint = chunked.get('system_fingerprint')+ process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "gpt-4o")-- # Calculate tokens using input prompt and aggregated response- input_tokens = general_tokens(prompt)- output_tokens = general_tokens(self._llmresponse)-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvetion.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_tokens", -1))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop", []))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- self._response_model)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- self._system_fingerprint)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ event_provider=event_provider,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._response_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)except Exception as e:handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)- finally:- self._span.end()raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat.completions' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.-- Returns:- The response from the original 'chat.completions' method.+ Wraps the GenAI function call."""- # Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)request_model = kwargs.get("model", "gpt-4o")@@ -294,292 +112,33 @@ def async_complete(version, environment, application_name,# pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = await wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)+ return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)- # Handling for non-streaming responseselse:- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = await wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('prompt_tokens')- output_tokens = response_dict.get('usage').get('completion_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop", []))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- response_dict.get('model'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- response_dict.get('system_fingerprint'))-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- for i in range(kwargs.get('n',1)):- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices')[i].get('finish_reason')])- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),- },- )- if kwargs.get('tools'):- span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,- str(response_dict.get('choices')[i].get('message').get('tool_calls')))-- if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "text")- elif response_dict.get('choices')[i].get('message').get('content') is not None:- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=response_dict.get('model'),- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def async_embedding(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for embeddings to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure Inference API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure Inference usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the embeddings method to add telemetry.- """-- async def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'embeddings' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'embeddings' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'embeddings' method.- kwargs: Keyword arguments for the 'embeddings' method.-- Returns:- The response from the original 'embeddings' method.- """-- server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)- request_model = kwargs.get("model", "text-embedding-ada-002")-- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"-- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:- start_time = time.time()- response = await wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)- try:- input_tokens = response_dict.get('usage').get('prompt_tokens')-- # Calculate cost of the operation- cost = get_embed_model_cost(request_model,- pricing_info, input_tokens)-- # Set Span attributes (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- [kwargs.get('encoding_format', 'float')])- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvetion.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)-- # Set Span attributes (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("input", "")),- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )- metrics["genai_client_usage_tokens"].record(- input_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ event_provider=event_provider,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.pyindex 14578044..81470d7b 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py@@ -4,18 +4,15 @@ Module for monitoring Azure AI Inference API calls.import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,- get_embed_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,set_server_address_and_port,- general_tokens+)+from openlit.instrumentation.azure_ai_inference.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,)from openlit.semcov import SemanticConvetion@@ -23,37 +20,21 @@ from openlit.semcov import SemanticConvetionlogger = logging.getLogger(__name__)def complete(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure AI Inference API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure AI Inference usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for GenAI function call"""class TracedSyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -61,12 +42,13 @@ def complete(version, environment, application_name,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""self._response_id = ""self._response_model = ""self._finish_reason = ""- self._system_fingerprint = ""+ self._input_tokens = 0+ self._output_tokens = 0self._args = argsself._kwargs = kwargs@@ -95,197 +77,33 @@ def complete(version, environment, application_name,def __next__(self):try:chunk = self.__wrapped__.__next__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from events- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and- 'content' in chunked.get('choices')[0].get('delta'))):-- content = chunked.get('choices')[0].get('delta').get('content')- if content:- self._llmresponse += content- self._response_id = chunked.get('id')- self._response_model = chunked.get('model')- self._finish_reason = chunked.get('choices')[0].get('finish_reason')- self._system_fingerprint = chunked.get('system_fingerprint')+ process_chunk(self, chunk)return chunkexcept StopIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "gpt-4o")-- # Calculate tokens using input prompt and aggregated response- input_tokens = general_tokens(prompt)- output_tokens = general_tokens(self._llmresponse)-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvetion.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_tokens", -1))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop", []))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- self._response_model)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- self._system_fingerprint)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ event_provider=event_provider,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._response_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)except Exception as e:handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)- finally:- self._span.end()raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat.completions' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.-- Returns:- The response from the original 'chat.completions' method.+ Wraps the GenAI function call."""- # Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)request_model = kwargs.get("model", "gpt-4o")@@ -294,292 +112,33 @@ def complete(version, environment, application_name,# pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)+ return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)- # Handling for non-streaming responseselse:- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('prompt_tokens')- output_tokens = response_dict.get('usage').get('completion_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop", []))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- response_dict.get('model'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- response_dict.get('system_fingerprint'))-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- for i in range(kwargs.get('n',1)):- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices')[i].get('finish_reason')])- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),- },- )- if kwargs.get('tools'):- span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,- str(response_dict.get('choices')[i].get('message').get('tool_calls')))-- if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "text")- elif response_dict.get('choices')[i].get('message').get('content') is not None:- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=response_dict.get('model'),- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def embedding(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for embeddings to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Azure Inference API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Azure Inference usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the embeddings method to add telemetry.- """-- def wrapper(wrapped, instance, args, kwargs):- """- Wraps the 'embeddings' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'embeddings' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'embeddings' method.- kwargs: Keyword arguments for the 'embeddings' method.-- Returns:- The response from the original 'embeddings' method.- """-- server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)- request_model = kwargs.get("model", "text-embedding-ada-002")-- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"-- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:- start_time = time.time()- response = wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)- try:- input_tokens = response_dict.get('usage').get('prompt_tokens')-- # Calculate cost of the operation- cost = get_embed_model_cost(request_model,- pricing_info, input_tokens)-- # Set Span attributes (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- [kwargs.get('encoding_format', 'float')])- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvetion.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)-- # Set Span attributes (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("input", "")),- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )- metrics["genai_client_usage_tokens"].record(- input_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ event_provider=event_provider,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.pynew file mode 100644index 00000000..5fc07078--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py@@ -0,0 +1,225 @@+"""+Azure AI Inference OpenTelemetry instrumentation utility functions+"""+import time++from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import Status, StatusCode++from openlit.__helpers import (+ calculate_ttft,+ response_as_dict,+ calculate_tbt,+ extract_and_format_input,+ get_chat_model_cost,+ create_metrics_attributes,+ otel_event,+ concatenate_all_contents+)+from openlit.semcov import SemanticConvetion++def process_chunk(self, chunk):+ """+ Process a chunk of response data and update state.+ """++ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)++ # Collect message IDs and aggregated response from events+ if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and+ 'content' in chunked.get('choices')[0].get('delta'))):++ if content := chunked.get('choices')[0].get('delta').get('content'):+ self._llmresponse += content++ if chunked.get('choices')[0].get('finish_reason') is not None:+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')++ if chunked.get('usage') is not None:+ self._input_tokens = chunked.get('usage').get('prompt_tokens')+ self._response_id = chunked.get('id')+ self._response_model = chunked.get('model')+ self._output_tokens = chunked.get('usage').get('completion_tokens')++def common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ event_provider, capture_message_content, disable_metrics, version, is_stream):+ """+ Process chat request and generate Telemetry+ """++ scope._end_time = time.time()+ if len(scope._timestamps) > 1:+ scope._tbt = calculate_tbt(scope._timestamps)++ formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))+ request_model = scope._kwargs.get('model', 'claude-3-opus-20240229')++ cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)++ # Set Span attributes (OTel Semconv)+ scope._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')+ scope._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION, SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM, SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL, request_model)+ scope._span.set_attribute(SemanticConvetion.SERVER_PORT, scope._server_port)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get('max_tokens', -1))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get('stop', []))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get('temperature', 1.0))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K, scope._kwargs.get('top_k', 1.0))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P, scope._kwargs.get('top_p', 1.0))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ scope._kwargs.get("frequency_penalty", 0.0))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ scope._kwargs.get("presence_penalty", 0.0))+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID, scope._response_id)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL, scope._response_model)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)+ scope._span.set_attribute(SemanticConvetion.SERVER_ADDRESS, scope._server_address)++ scope._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text' if isinstance(scope._llmresponse, str) else 'json')++ scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)+ scope._span.set_attribute(SERVICE_NAME, application_name)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM, is_stream)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST, cost)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT, scope._tbt)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT, scope._ttft)+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION, version)++ # To be removed one the change to log events (from span events) is complete+ prompt = concatenate_all_contents(formatted_messages)+ if capture_message_content:+ scope._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ scope._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,+ },+ )++ choice_event_body = {+ 'finish_reason': scope._finish_reason,+ 'index': 0,+ 'message': {+ **({'content': scope._llmresponse} if capture_message_content else {}),+ 'role': 'assistant'+ }+ }++ # Emit events+ for role in ['user', 'system', 'assistant', 'tool']:+ if formatted_messages.get(role, {}).get('content', ''):+ event = otel_event(+ name=getattr(SemanticConvetion, f'GEN_AI_{role.upper()}_MESSAGE'),+ attributes={+ SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE+ },+ body = {+ # pylint: disable=line-too-long+ **({'content': formatted_messages.get(role, {}).get('content', '')} if capture_message_content else {}),+ 'role': formatted_messages.get(role, {}).get('role', []),+ **({+ 'tool_calls': {+ 'function': {+ # pylint: disable=line-too-long+ 'name': (scope._tool_calls[0].get('function', {}).get('name', '') if scope._tool_calls else ''),+ 'arguments': (scope._tool_calls[0].get('function', {}).get('arguments', '') if scope._tool_calls else '')+ },+ 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else ''),+ 'type': 'function'+ }+ } if role == 'assistant' else {}),+ **({+ 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else '')+ } if role == 'tool' else {})+ }+ )+ event_provider.emit(event)++ choice_event = otel_event(+ name=SemanticConvetion.GEN_AI_CHOICE,+ attributes={+ SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE+ },+ body=choice_event_body+ )+ event_provider.emit(choice_event)++ scope._span.set_status(Status(StatusCode.OK))++ if not disable_metrics:+ metrics_attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ request_model=request_model,+ server_address=scope._server_address,+ server_port=scope._server_port,+ response_model=scope._response_model,+ )++ metrics['genai_client_usage_tokens'].record(scope._input_tokens + scope._output_tokens, metrics_attributes)+ metrics['genai_client_operation_duration'].record(scope._end_time - scope._start_time, metrics_attributes)+ metrics['genai_server_tbt'].record(scope._tbt, metrics_attributes)+ metrics['genai_server_ttft'].record(scope._ttft, metrics_attributes)+ metrics['genai_requests'].add(1, metrics_attributes)+ metrics['genai_completion_tokens'].add(scope._output_tokens, metrics_attributes)+ metrics['genai_prompt_tokens'].add(scope._input_tokens, metrics_attributes)+ metrics['genai_cost'].record(cost, metrics_attributes)++def process_streaming_chat_response(self, pricing_info, environment, application_name, metrics,+ event_provider, capture_message_content=False, disable_metrics=False, version=''):+ """+ Process chat request and generate Telemetry+ """++ common_chat_logic(self, pricing_info, environment, application_name, metrics,+ event_provider, capture_message_content, disable_metrics, version, is_stream=True)++def process_chat_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, event_provider, start_time,+ span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):+ """+ Process chat request and generate Telemetry+ """++ self = type('GenericScope', (), {})()+ response_dict = response_as_dict(response)++ # pylint: disable = no-member+ self._start_time = start_time+ self._end_time = time.time()+ self._span = span+ self._llmresponse = response_dict.get('choices', {})[0].get('message', '').get('content', '')+ self._input_tokens = response_dict.get('usage').get('prompt_tokens')+ self._output_tokens = response_dict.get('usage').get('completion_tokens')+ self._response_model = response_dict.get('model', '')+ self._finish_reason = response_dict.get('choices', {})[0].get('finish_reason', '')+ self._response_id = response_dict.get('id', '')+ self._timestamps = []+ self._ttft, self._tbt = self._end_time - self._start_time, 0+ self._server_address, self._server_port = server_address, server_port+ self._kwargs = kwargs++ common_chat_logic(self, pricing_info, environment, application_name, metrics,+ event_provider, capture_message_content, disable_metrics, version, is_stream=False)++ return response