Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Refactor Azure Inference OpenTelemetry Monitoring Instrumentation (#779)
Patcher2c5747315745
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 01843217..0a15583b 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.34.15"+version = "1.34.16"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.pyindex b4ea879c..45a26408 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py@@ -5,49 +5,65 @@ import importlib.metadatafrom opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapperfrom openlit.instrumentation.azure_ai_inference.azure_ai_inference import (- complete+ complete,+ embed)from openlit.instrumentation.azure_ai_inference.async_azure_ai_inference import (- async_complete+ async_complete,+ async_embed)-_instruments = ('azure-ai-inference >= 1.0.0b4',)+_instruments = ("azure-ai-inference >= 1.0.0b4",)class AzureAIInferenceInstrumentor(BaseInstrumentor):"""- An instrumentor for azure-ai-inference's client library.+ An instrumentor for azure-ai-inference client library."""def instrumentation_dependencies(self) -> Collection[str]:return _instrumentsdef _instrument(self, **kwargs):- application_name = kwargs.get('application_name', 'default')- environment = kwargs.get('environment', 'default')- tracer = kwargs.get('tracer')- event_provider = kwargs.get('event_provider')- metrics = kwargs.get('metrics_dict')- pricing_info = kwargs.get('pricing_info', {})- capture_message_content = kwargs.get('capture_message_content', False)- disable_metrics = kwargs.get('disable_metrics')- version = importlib.metadata.version('azure-ai-inference')-- # sync generate+ application_name = kwargs.get("application_name", "default")+ environment = kwargs.get("environment", "default")+ tracer = kwargs.get("tracer")+ metrics = kwargs.get("metrics_dict")+ pricing_info = kwargs.get("pricing_info", {})+ capture_message_content = kwargs.get("capture_message_content", False)+ disable_metrics = kwargs.get("disable_metrics")+ version = importlib.metadata.version("azure-ai-inference")++ # sync chat completionswrap_function_wrapper(- 'azure.ai.inference',- 'ChatCompletionsClient.complete',+ "azure.ai.inference",+ "ChatCompletionsClient.complete",complete(version, environment, application_name,- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)- # async generate+ # async chat completionswrap_function_wrapper(- 'azure.ai.inference.aio',- 'ChatCompletionsClient.complete',+ "azure.ai.inference.aio",+ "ChatCompletionsClient.complete",async_complete(version, environment, application_name,- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ )++ # sync embeddings+ wrap_function_wrapper(+ "azure.ai.inference",+ "EmbeddingsClient.embed",+ embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ )++ # async embeddings+ wrap_function_wrapper(+ "azure.ai.inference.aio",+ "EmbeddingsClient.embed",+ async_embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)def _uninstrument(self, **kwargs):- # Proper uninstrumentation logic to revert patched methodspassdiff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.pyindex d860e522..9a87b8bc 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py@@ -13,6 +13,7 @@ from openlit.instrumentation.azure_ai_inference.utils import (process_chunk,process_chat_response,process_streaming_chat_response,+ process_embedding_response,)from openlit.semcov import SemanticConvention@@ -20,7 +21,7 @@ from openlit.semcov import SemanticConventionlogger = logging.getLogger(__name__)def async_complete(version, environment, application_name,- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""Generates a telemetry wrapper for GenAI function call"""@@ -43,12 +44,15 @@ def async_complete(version, environment, application_name,self.__wrapped__ = wrappedself._span = spanself._span_name = span_name- self._llmresponse = ''- self._response_id = ''- self._response_model = ''- self._finish_reason = ''+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._response_service_tier = ""+ self._tools = Noneself._input_tokens = 0self._output_tokens = 0+ self._reasoning_tokens = 0self._args = argsself._kwargs = kwargs@@ -64,53 +68,35 @@ def async_complete(version, environment, application_name,await self.__wrapped__.__aenter__()return self- async def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ async def __aexit__(self, exc_type, exc_val, exc_tb):+ await self.__wrapped__.__aexit__(exc_type, exc_val, exc_tb)+ process_streaming_chat_response(+ self, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version+ )def __aiter__(self):return self- async def __getattr__(self, name):- """Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)-async def __anext__(self):- try:- chunk = await self.__wrapped__.__anext__()- process_chunk(self, chunk)- return chunk- except StopAsyncIteration:- try:- with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- event_provider=event_provider,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version- )-- except Exception as e:- handle_exception(self._span, e)- logger.error('Error in trace creation: %s', e)- raise+ chunk = await self.__wrapped__.__anext__()+ process_chunk(self, chunk)+ return chunk++ def __getattr__(self, name):+ return getattr(self.__wrapped__, name)async def wrapper(wrapped, instance, args, kwargs):"""Wraps the GenAI function call."""- streaming = kwargs.get('stream', False)- server_address, server_port = set_server_address_and_port(instance, 'models.github.ai', 443)- request_model = kwargs.get('model', 'gpt-4o')+ streaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "gpt-4o")- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:awaited_wrapped = await wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)@@ -130,7 +116,6 @@ def async_complete(version, environment, application_name,environment=environment,application_name=application_name,metrics=metrics,- event_provider=event_provider,start_time=start_time,span=span,capture_message_content=capture_message_content,@@ -142,3 +127,48 @@ def async_complete(version, environment, application_name,return responsereturn wrapper++def async_embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for GenAI embedding function call+ """++ async def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps the GenAI embedding function call.+ """++ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "text-embedding-3-small")++ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = await wrapped(*args, **kwargs)++ try:+ response = process_embedding_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ except Exception as e:+ handle_exception(span, e)++ return response++ return wrapperdiff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.pyindex 1e5cf519..23baa88e 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py@@ -13,6 +13,7 @@ from openlit.instrumentation.azure_ai_inference.utils import (process_chunk,process_chat_response,process_streaming_chat_response,+ process_embedding_response,)from openlit.semcov import SemanticConvention@@ -20,7 +21,7 @@ from openlit.semcov import SemanticConventionlogger = logging.getLogger(__name__)def complete(version, environment, application_name,- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""Generates a telemetry wrapper for GenAI function call"""@@ -43,12 +44,15 @@ def complete(version, environment, application_name,self.__wrapped__ = wrappedself._span = spanself._span_name = span_name- self._llmresponse = ''- self._response_id = ''- self._response_model = ''- self._finish_reason = ''+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._response_service_tier = ""+ self._tools = Noneself._input_tokens = 0self._output_tokens = 0+ self._reasoning_tokens = 0self._args = argsself._kwargs = kwargs@@ -64,53 +68,35 @@ def complete(version, environment, application_name,self.__wrapped__.__enter__()return self- def __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ def __exit__(self, exc_type, exc_val, exc_tb):+ self.__wrapped__.__exit__(exc_type, exc_val, exc_tb)+ process_streaming_chat_response(+ self, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version+ )def __iter__(self):return self+ def __next__(self):+ chunk = next(self.__wrapped__)+ process_chunk(self, chunk)+ return chunk+def __getattr__(self, name):- """Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)- def __next__(self):- try:- chunk = self.__wrapped__.__next__()- process_chunk(self, chunk)- return chunk- except StopIteration:- try:- with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- event_provider=event_provider,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version- )-- except Exception as e:- handle_exception(self._span, e)- logger.error('Error in trace creation: %s', e)- raise-def wrapper(wrapped, instance, args, kwargs):"""Wraps the GenAI function call."""- streaming = kwargs.get('stream', False)- server_address, server_port = set_server_address_and_port(instance, 'models.github.ai', 443)- request_model = kwargs.get('model', 'gpt-4o')+ streaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "gpt-4o")- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:awaited_wrapped = wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)@@ -130,7 +116,6 @@ def complete(version, environment, application_name,environment=environment,application_name=application_name,metrics=metrics,- event_provider=event_provider,start_time=start_time,span=span,capture_message_content=capture_message_content,@@ -142,3 +127,48 @@ def complete(version, environment, application_name,return responsereturn wrapper++def embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for GenAI embedding function call+ """++ def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps the GenAI embedding function call.+ """++ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)+ request_model = kwargs.get("model", "text-embedding-3-small")++ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)++ try:+ response = process_embedding_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ except Exception as e:+ handle_exception(span, e)++ return response++ return wrapperdiff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.pyindex f45fb62d..321e5725 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py@@ -3,104 +3,185 @@ Azure AI Inference OpenTelemetry instrumentation utility functions"""import time-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom opentelemetry.trace import Status, StatusCodefrom openlit.__helpers import (calculate_ttft,response_as_dict,calculate_tbt,- extract_and_format_input,get_chat_model_cost,- create_metrics_attributes,- otel_event,- concatenate_all_contents+ get_embed_model_cost,+ common_span_attributes,+ record_completion_metrics,+ record_embedding_metrics,)from openlit.semcov import SemanticConvention-def process_chunk(self, chunk):+def format_content(messages):+ """+ Process a list of messages to extract content.+ """++ formatted_messages = []+ for message in messages:+ role = message.get("role", "user")+ content = message.get("content", "")++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item.get("image_url", "")}'+ if "type" in item else f'text: {item.get("text", "")}'+ for item in content+ )+ formatted_messages.append(f'{role}: {content_str}')+ else:+ formatted_messages.append(f'{role}: {content}')++ return '\n'.join(formatted_messages)++def process_chunk(scope, chunk):"""Process a chunk of response data and update state."""end_time = time.time()# Record the timestamp for the current chunk- self._timestamps.append(end_time)+ scope._timestamps.append(end_time)- if len(self._timestamps) == 1:+ if len(scope._timestamps) == 1:# Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)+ scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)chunked = response_as_dict(chunk)# Collect message IDs and aggregated response from events- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and- 'content' in chunked.get('choices')[0].get('delta'))):+ choices = chunked.get("choices", [])+ if choices and "delta" in choices[0]:+ delta = choices[0]["delta"]++ # Handle content+ content = delta.get("content")+ if content:+ scope._llmresponse += content++ # Handle reasoning content (if present)+ reasoning_content = delta.get("reasoning_content")+ if reasoning_content:+ if not hasattr(scope, "_reasoning_content"):+ scope._reasoning_content = ""+ scope._reasoning_content += reasoning_content- if content := chunked.get('choices')[0].get('delta').get('content'):- self._llmresponse += content+ # Handle finish_reason (appears in final chunk)+ finish_reason = chunked.get("choices")[0].get("finish_reason")+ if finish_reason:+ scope._finish_reason = finish_reason+ scope._end_time = time.time()- if chunked.get('choices')[0].get('finish_reason') is not None:- self._finish_reason = chunked.get('choices')[0].get('finish_reason')+ # Handle tool calls in streaming - optimized+ delta_tools = delta.get("tool_calls")+ if delta_tools:+ scope._tools = scope._tools or []- if chunked.get('usage') is not None:- self._input_tokens = chunked.get('usage').get('prompt_tokens')- self._response_id = chunked.get('id')- self._response_model = chunked.get('model')- self._output_tokens = chunked.get('usage').get('completion_tokens')+ for tool in delta_tools:+ idx = tool.get("index", 0)++ # Extend list if needed+ scope._tools.extend([{}] * (idx + 1 - len(scope._tools)))++ if tool.get("id"): # New tool (id exists)+ func = tool.get("function", {})+ scope._tools[idx] = {+ "id": tool["id"],+ "function": {"name": func.get("name", ""), "arguments": func.get("arguments", "")},+ "type": tool.get("type", "function")+ }+ elif scope._tools[idx] and "function" in tool: # Append args (id is None)+ scope._tools[idx]["function"]["arguments"] += tool["function"].get("arguments", "")++ # Handle usage information (typically only in final chunk)+ if chunked.get("usage"):+ scope._input_tokens = chunked.get("usage").get("prompt_tokens", 0)+ scope._output_tokens = chunked.get("usage").get("completion_tokens", 0)+ # Handle reasoning tokens if present (optional) - check nested structure+ completion_details = chunked.get("usage", {}).get("completion_tokens_details", {})+ if "reasoning_tokens" in completion_details:+ scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)+ elif "reasoning_tokens" in chunked.get("usage", {}):+ scope._reasoning_tokens = chunked.get("usage").get("reasoning_tokens", 0)def common_chat_logic(scope, pricing_info, environment, application_name, metrics,- event_provider, capture_message_content, disable_metrics, version, is_stream):+ capture_message_content, disable_metrics, version, is_stream):"""Process chat request and generate Telemetry"""- scope._end_time = time.time()if len(scope._timestamps) > 1:scope._tbt = calculate_tbt(scope._timestamps)- formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))- request_model = scope._kwargs.get('model', 'claude-3-opus-20240229')+ prompt = format_content(scope._kwargs.get("messages", []))+ request_model = scope._kwargs.get("model", "gpt-4o")cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)- # Set Span attributes (OTel Semconv)- scope._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')- scope._span.set_attribute(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- scope._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL, request_model)- scope._span.set_attribute(SemanticConvention.SERVER_PORT, scope._server_port)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get('max_tokens', -1))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get('stop', []))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get('temperature', 1.0))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, scope._kwargs.get('top_k', 1.0))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get('top_p', 1.0))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- scope._kwargs.get('frequency_penalty', 0.0))- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- scope._kwargs.get('presence_penalty', 0.0))- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])+ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ scope._server_address, scope._server_port, request_model, scope._response_model,+ environment, application_name, is_stream, scope._tbt, scope._ttft, version)++ # Span Attributes for Request parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, scope._kwargs.get("frequency_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get("max_tokens", -1))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, scope._kwargs.get("presence_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get("stop", []))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get("temperature", 1.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get("top_p", 1.0))++ # Span Attributes for Response parametersscope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, scope._response_id)- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, scope._response_model)+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SERVICE_TIER, scope._response_service_tier)+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT, scope._response_service_tier)+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text" if isinstance(scope._llmresponse, str) else "json")++ # Span Attributes for Cost and Tokensscope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)- scope._span.set_attribute(SemanticConvention.SERVER_ADDRESS, scope._server_address)-- scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- 'text' if isinstance(scope._llmresponse, str) else 'json')-- scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)- scope._span.set_attribute(SERVICE_NAME, application_name)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, is_stream)scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT, scope._tbt)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT, scope._ttft)- scope._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, version)- # To be removed one the change to log events (from span events) is complete- prompt = concatenate_all_contents(formatted_messages)+ # Span Attributes for Reasoning (if present)+ if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens)+ # Update total token usage to include reasoning tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,+ scope._input_tokens + scope._output_tokens + scope._reasoning_tokens)++ # Span Attributes for Tools - optimized+ if scope._tools:+ tools = scope._tools if isinstance(scope._tools, list) else [scope._tools]++ names, ids, args = zip(*[+ (t.get("function", {}).get("name", ""),+ str(t.get("id", "")),+ str(t.get("function", {}).get("arguments", "")))+ for t in tools if isinstance(t, dict) and t+ ]) if tools else ([], [], [])++ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, ", ".join(filter(None, names)))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, ", ".join(filter(None, ids)))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, ", ".join(filter(None, args)))++ # Span Attributes for Contentif capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse)++ # Add reasoning content if available+ if hasattr(scope, "_reasoning_content") and scope._reasoning_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_REASONING, scope._reasoning_content)++ # To be removed once the change to span_attributes (from span events) is completescope._span.add_event(name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,attributes={@@ -114,112 +195,143 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric},)- choice_event_body = {- 'finish_reason': scope._finish_reason,- 'index': 0,- 'message': {- **({'content': scope._llmresponse} if capture_message_content else {}),- 'role': 'assistant'- }- }-- # Emit events- for role in ['user', 'system', 'assistant', 'tool']:- if formatted_messages.get(role, {}).get('content', ''):- event = otel_event(- name=getattr(SemanticConvention, f'GEN_AI_{role.upper()}_MESSAGE'),- attributes={- SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE- },- body = {- # pylint: disable=line-too-long- **({'content': formatted_messages.get(role, {}).get('content', '')} if capture_message_content else {}),- 'role': formatted_messages.get(role, {}).get('role', []),- **({- 'tool_calls': {- 'function': {- # pylint: disable=line-too-long- 'name': (scope._tool_calls[0].get('function', {}).get('name', '') if scope._tool_calls else ''),- 'arguments': (scope._tool_calls[0].get('function', {}).get('arguments', '') if scope._tool_calls else '')- },- 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else ''),- 'type': 'function'- }- } if role == 'assistant' else {}),- **({- 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else '')- } if role == 'tool' else {})- }- )- event_provider.emit(event)-- choice_event = otel_event(- name=SemanticConvention.GEN_AI_CHOICE,- attributes={- SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE- },- body=choice_event_body- )- event_provider.emit(choice_event)-scope._span.set_status(Status(StatusCode.OK))+ # Metricsif not disable_metrics:- metrics_attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,- request_model=request_model,- server_address=scope._server_address,- server_port=scope._server_port,- response_model=scope._response_model,- )+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,+ SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE, scope._server_address, scope._server_port,+ request_model, scope._response_model, environment, application_name, scope._start_time, scope._end_time,+ scope._input_tokens, scope._output_tokens, cost, scope._tbt, scope._ttft)- metrics['genai_client_usage_tokens'].record(scope._input_tokens + scope._output_tokens, metrics_attributes)- metrics['genai_client_operation_duration'].record(scope._end_time - scope._start_time, metrics_attributes)- metrics['genai_server_tbt'].record(scope._tbt, metrics_attributes)- metrics['genai_server_ttft'].record(scope._ttft, metrics_attributes)- metrics['genai_requests'].add(1, metrics_attributes)- metrics['genai_completion_tokens'].add(scope._output_tokens, metrics_attributes)- metrics['genai_prompt_tokens'].add(scope._input_tokens, metrics_attributes)- metrics['genai_cost'].record(cost, metrics_attributes)--def process_streaming_chat_response(self, pricing_info, environment, application_name, metrics,- event_provider, capture_message_content=False, disable_metrics=False, version=''):+def process_streaming_chat_response(scope, pricing_info, environment, application_name, metrics,+ capture_message_content=False, disable_metrics=False, version=""):"""- Process chat request and generate Telemetry+ Process streaming chat request and generate Telemetry"""- common_chat_logic(self, pricing_info, environment, application_name, metrics,- event_provider, capture_message_content, disable_metrics, version, is_stream=True)+ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=True)def process_chat_response(response, request_model, pricing_info, server_port, server_address,- environment, application_name, metrics, event_provider, start_time,- span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):+ environment, application_name, metrics, start_time, span, capture_message_content=False,+ disable_metrics=False, version="1.0.0", **kwargs):"""Process chat request and generate Telemetry"""- self = type('GenericScope', (), {})()+ # Create scope object+ scope = type("GenericScope", (), {})()+ response_dict = response_as_dict(response)++ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ scope._llmresponse = " ".join(+ (choice.get("message", {}).get("content") or "")+ for choice in response_dict.get("choices", [])+ )+ # Handle reasoning content from non-streaming response+ reasoning_content = response_dict.get("choices", [{}])[0].get("message", {}).get("reasoning_content")+ if reasoning_content:+ scope._reasoning_content = reasoning_content++ scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)+ scope._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)+ # Handle reasoning tokens if present (optional) - check nested structure+ completion_details = response_dict.get("usage", {}).get("completion_tokens_details", {})+ if "reasoning_tokens" in completion_details:+ scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)+ elif "reasoning_tokens" in response_dict.get("usage", {}):+ scope._reasoning_tokens = response_dict.get("usage").get("reasoning_tokens", 0)+ else:+ scope._reasoning_tokens = 0+ scope._response_id = response_dict.get("id")+ scope._response_model = response_dict.get("model")+ scope._finish_reason = str(response_dict.get("choices", [])[0].get("finish_reason", ""))+ scope._response_service_tier = str(response_dict.get("system_fingerprint", ""))+ scope._timestamps = []+ scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0+ scope._server_address, scope._server_port = server_address, server_port+ scope._kwargs = kwargs++ # Handle tool calls+ if scope._kwargs.get("tools"):+ scope._tools = response_dict.get("choices", [{}])[0].get("message", {}).get("tool_calls")+ else:+ scope._tools = None++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=False)++ return response++def common_embedding_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version):+ """+ Process embedding request and generate Telemetry+ """++ request_model = scope._kwargs.get("model", "text-embedding-3-small")++ cost = get_embed_model_cost(request_model, pricing_info, scope._input_tokens)++ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,+ scope._server_address, scope._server_port, request_model, scope._response_model,+ environment, application_name, False, 0, scope._end_time - scope._start_time, version)++ # Span Attributes for Request parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS, [scope._kwargs.get("encoding_format", "float")])+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER, scope._kwargs.get("user", ""))++ # Span Attributes for Cost and Tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)++ # Span Attributes for Content+ if capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, str(scope._kwargs.get("input", "")))++ # To be removed once the change to span_attributes (from span events) is complete+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_PROMPT: str(scope._kwargs.get("input", "")),+ },+ )++ scope._span.set_status(Status(StatusCode.OK))++ # Metrics+ if not disable_metrics:+ record_embedding_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING,+ SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE, scope._server_address, scope._server_port,+ request_model, scope._response_model, environment, application_name, scope._start_time, scope._end_time,+ scope._input_tokens, cost)++def process_embedding_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, start_time, span, capture_message_content=False,+ disable_metrics=False, version="1.0.0", **kwargs):+ """+ Process embedding request and generate Telemetry+ """++ # Create scope object+ scope = type("GenericScope", (), {})()response_dict = response_as_dict(response)- # pylint: disable = no-member- self._start_time = start_time- self._end_time = time.time()- self._span = span- self._llmresponse = response_dict.get('choices', {})[0].get('message', '').get('content', '')- self._input_tokens = response_dict.get('usage').get('prompt_tokens')- self._output_tokens = response_dict.get('usage').get('completion_tokens')- self._response_model = response_dict.get('model', '')- self._finish_reason = response_dict.get('choices', {})[0].get('finish_reason', '')- self._response_id = response_dict.get('id', '')- self._timestamps = []- self._ttft, self._tbt = self._end_time - self._start_time, 0- self._server_address, self._server_port = server_address, server_port- self._kwargs = kwargs-- common_chat_logic(self, pricing_info, environment, application_name, metrics,- event_provider, capture_message_content, disable_metrics, version, is_stream=False)+ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)+ scope._response_model = response_dict.get("model")+ scope._server_address, scope._server_port = server_address, server_port+ scope._kwargs = kwargs++ common_embedding_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version)return responsediff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex d4b04b59..e0cc8b82 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -166,6 +166,7 @@ class SemanticConvention:GEN_AI_CONTENT_COMPLETION_EVENT = "gen_ai.content.completion"GEN_AI_CONTENT_COMPLETION = "gen_ai.completion"GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt"+ GEN_AI_CONTENT_REASONING = "gen_ai.content.reasoning"# GenAI RagGEN_AI_RAG_MAX_SEGMENTS = "gen_ai.rag.max_segments"diff --git a/sdk/python/tests/test_azure_ai_inference.py.hold b/sdk/python/tests/test_azure_ai_inference.pysimilarity index 81%rename from sdk/python/tests/test_azure_ai_inference.py.holdrename to sdk/python/tests/test_azure_ai_inference.pyindex d137a820..7cd80743 100644--- a/sdk/python/tests/test_azure_ai_inference.py.hold+++ b/sdk/python/tests/test_azure_ai_inference.py@@ -24,13 +24,13 @@ from azure.core.credentials import AzureKeyCredential# Initialize synchronous Azure AI Inference Chat clientsync_chat_client = ChatCompletionsClient(- endpoint="https://models.inference.ai.azure.com",+ endpoint="https://models.github.ai/inference",credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),)# Initialize synchronous Azure AI Inference Chat clientsync_embed_client = EmbeddingsClient(- endpoint="https://models.inference.ai.azure.com",+ endpoint="https://models.github.ai/inference",credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),)@@ -39,7 +39,7 @@ openlit.init(environment="openlit-testing", application_name="openlit-python-tesdef test_sync_chat_completions():"""- Tests synchronous chat completions with the "Phi-3-mini-4k-instruct" model.+ Tests synchronous chat completions with the "microsoft/Phi-4-mini-reasoning" model.Raises:AssertionError: If the chat completion response object is not as expected.@@ -50,7 +50,7 @@ def test_sync_chat_completions():SystemMessage(content="You are a helpful assistant."),UserMessage(content="sync non-streaming"),],- model="Phi-3-mini-4k-instruct",+ model="microsoft/Phi-4-mini-reasoning",temperature=1.0,max_tokens=1000,top_p=1.0@@ -59,7 +59,7 @@ def test_sync_chat_completions():def test_sync_embeddings():"""- Tests synchronous embedding creation with the "cohere-embed-v3-english" model.+ Tests synchronous embedding creation with the "openai/text-embedding-3-small" model.Raises:AssertionError: If the embedding response object is not as expected.@@ -67,25 +67,25 @@ def test_sync_embeddings():response = sync_embed_client.embed(input=["LLM Observability", "Monitor GPUs"],- model="cohere-embed-v3-english"+ model="openai/text-embedding-3-small")assert response.data[0]["object"] == 'embedding'from azure.ai.inference.aio import ChatCompletionsClient, EmbeddingsClientasync_chat_client = ChatCompletionsClient(- endpoint="https://models.inference.ai.azure.com",+ endpoint="https://models.github.ai/inference",credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),)async_embed_client = EmbeddingsClient(- endpoint="https://models.inference.ai.azure.com",+ endpoint="https://models.github.ai/inference",credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),)@pytest.mark.asyncioasync def test_async_chat_completions():"""- Tests synchronous chat completions with the "Phi-3-mini-4k-instruct" model.+ Tests synchronous chat completions with the "microsoft/Phi-4-mini-reasoning" model.Raises:AssertionError: If the chat completion response object is not as expected.@@ -96,7 +96,7 @@ async def test_async_chat_completions():SystemMessage(content="You are a helpful assistant."),UserMessage(content="sync non-streaming"),],- model="Phi-3-mini-4k-instruct",+ model="microsoft/Phi-4-mini-reasoning",temperature=1.0,max_tokens=1000,top_p=1.0@@ -106,7 +106,7 @@ async def test_async_chat_completions():@pytest.mark.asyncioasync def test_async_embeddings():"""- Tests synchronous embedding creation with the "cohere-embed-v3-english" model.+ Tests synchronous embedding creation with the "openai/text-embedding-3-small" model.Raises:AssertionError: If the embedding response object is not as expected.@@ -114,6 +114,6 @@ async def test_async_embeddings():response = await async_embed_client.embed(input=["LLM Observability", "Monitor GPUs"],- model="cohere-embed-v3-english"+ model="openai/text-embedding-3-small")assert response.data[0]["object"] == 'embedding'