Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Refactor Azure Inference OpenTelemetry Monitoring Instrumentation (#779)

Patcher2c5747315745
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 01843217..0a15583b 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.34.15"
+version = "1.34.16"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py
index b4ea879c..45a26408 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/__init__.py
@@ -5,49 +5,65 @@ import importlib.metadata
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from wrapt import wrap_function_wrapper
from openlit.instrumentation.azure_ai_inference.azure_ai_inference import (
- complete
+ complete,
+ embed
)
from openlit.instrumentation.azure_ai_inference.async_azure_ai_inference import (
- async_complete
+ async_complete,
+ async_embed
)
-_instruments = ('azure-ai-inference >= 1.0.0b4',)
+_instruments = ("azure-ai-inference >= 1.0.0b4",)
class AzureAIInferenceInstrumentor(BaseInstrumentor):
"""
- An instrumentor for azure-ai-inference's client library.
+ An instrumentor for azure-ai-inference client library.
"""
def instrumentation_dependencies(self) -> Collection[str]:
return _instruments
def _instrument(self, **kwargs):
- application_name = kwargs.get('application_name', 'default')
- environment = kwargs.get('environment', 'default')
- tracer = kwargs.get('tracer')
- event_provider = kwargs.get('event_provider')
- metrics = kwargs.get('metrics_dict')
- pricing_info = kwargs.get('pricing_info', {})
- capture_message_content = kwargs.get('capture_message_content', False)
- disable_metrics = kwargs.get('disable_metrics')
- version = importlib.metadata.version('azure-ai-inference')
-
- # sync generate
+ application_name = kwargs.get("application_name", "default")
+ environment = kwargs.get("environment", "default")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ capture_message_content = kwargs.get("capture_message_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("azure-ai-inference")
+
+ # sync chat completions
wrap_function_wrapper(
- 'azure.ai.inference',
- 'ChatCompletionsClient.complete',
+ "azure.ai.inference",
+ "ChatCompletionsClient.complete",
complete(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
- # async generate
+ # async chat completions
wrap_function_wrapper(
- 'azure.ai.inference.aio',
- 'ChatCompletionsClient.complete',
+ "azure.ai.inference.aio",
+ "ChatCompletionsClient.complete",
async_complete(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
+ )
+
+ # sync embeddings
+ wrap_function_wrapper(
+ "azure.ai.inference",
+ "EmbeddingsClient.embed",
+ embed(version, environment, application_name,
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
+ )
+
+ # async embeddings
+ wrap_function_wrapper(
+ "azure.ai.inference.aio",
+ "EmbeddingsClient.embed",
+ async_embed(version, environment, application_name,
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
def _uninstrument(self, **kwargs):
- # Proper uninstrumentation logic to revert patched methods
pass
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
index d860e522..9a87b8bc 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py
@@ -13,6 +13,7 @@ from openlit.instrumentation.azure_ai_inference.utils import (
process_chunk,
process_chat_response,
process_streaming_chat_response,
+ process_embedding_response,
)
from openlit.semcov import SemanticConvention
@@ -20,7 +21,7 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
def async_complete(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -43,12 +44,15 @@ def async_complete(version, environment, application_name,
self.__wrapped__ = wrapped
self._span = span
self._span_name = span_name
- self._llmresponse = ''
- self._response_id = ''
- self._response_model = ''
- self._finish_reason = ''
+ self._llmresponse = ""
+ self._response_id = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._response_service_tier = ""
+ self._tools = None
self._input_tokens = 0
self._output_tokens = 0
+ self._reasoning_tokens = 0
self._args = args
self._kwargs = kwargs
@@ -64,53 +68,35 @@ def async_complete(version, environment, application_name,
await self.__wrapped__.__aenter__()
return self
- async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ await self.__wrapped__.__aexit__(exc_type, exc_val, exc_tb)
+ process_streaming_chat_response(
+ self, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version
+ )
def __aiter__(self):
return self
- async def __getattr__(self, name):
- """Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
-
async def __anext__(self):
- try:
- chunk = await self.__wrapped__.__anext__()
- process_chunk(self, chunk)
- return chunk
- except StopAsyncIteration:
- try:
- with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version
- )
-
- except Exception as e:
- handle_exception(self._span, e)
- logger.error('Error in trace creation: %s', e)
- raise
+ chunk = await self.__wrapped__.__anext__()
+ process_chunk(self, chunk)
+ return chunk
+
+ def __getattr__(self, name):
+ return getattr(self.__wrapped__, name)
async def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the GenAI function call.
"""
- streaming = kwargs.get('stream', False)
- server_address, server_port = set_server_address_and_port(instance, 'models.github.ai', 443)
- request_model = kwargs.get('model', 'gpt-4o')
+ streaming = kwargs.get("stream", False)
+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)
+ request_model = kwargs.get("model", "gpt-4o")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
- # pylint: disable=no-else-return
if streaming:
awaited_wrapped = await wrapped(*args, **kwargs)
span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
@@ -130,7 +116,6 @@ def async_complete(version, environment, application_name,
environment=environment,
application_name=application_name,
metrics=metrics,
- event_provider=event_provider,
start_time=start_time,
span=span,
capture_message_content=capture_message_content,
@@ -142,3 +127,48 @@ def async_complete(version, environment, application_name,
return response
return wrapper
+
+def async_embed(version, environment, application_name,
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for GenAI embedding function call
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the GenAI embedding function call.
+ """
+
+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)
+ request_model = kwargs.get("model", "text-embedding-3-small")
+
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ response = process_embedding_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
index 1e5cf519..23baa88e 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py
@@ -13,6 +13,7 @@ from openlit.instrumentation.azure_ai_inference.utils import (
process_chunk,
process_chat_response,
process_streaming_chat_response,
+ process_embedding_response,
)
from openlit.semcov import SemanticConvention
@@ -20,7 +21,7 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
def complete(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -43,12 +44,15 @@ def complete(version, environment, application_name,
self.__wrapped__ = wrapped
self._span = span
self._span_name = span_name
- self._llmresponse = ''
- self._response_id = ''
- self._response_model = ''
- self._finish_reason = ''
+ self._llmresponse = ""
+ self._response_id = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._response_service_tier = ""
+ self._tools = None
self._input_tokens = 0
self._output_tokens = 0
+ self._reasoning_tokens = 0
self._args = args
self._kwargs = kwargs
@@ -64,53 +68,35 @@ def complete(version, environment, application_name,
self.__wrapped__.__enter__()
return self
- def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.__wrapped__.__exit__(exc_type, exc_val, exc_tb)
+ process_streaming_chat_response(
+ self, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version
+ )
def __iter__(self):
return self
+ def __next__(self):
+ chunk = next(self.__wrapped__)
+ process_chunk(self, chunk)
+ return chunk
+
def __getattr__(self, name):
- """Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
- def __next__(self):
- try:
- chunk = self.__wrapped__.__next__()
- process_chunk(self, chunk)
- return chunk
- except StopIteration:
- try:
- with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version
- )
-
- except Exception as e:
- handle_exception(self._span, e)
- logger.error('Error in trace creation: %s', e)
- raise
-
def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the GenAI function call.
"""
- streaming = kwargs.get('stream', False)
- server_address, server_port = set_server_address_and_port(instance, 'models.github.ai', 443)
- request_model = kwargs.get('model', 'gpt-4o')
+ streaming = kwargs.get("stream", False)
+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)
+ request_model = kwargs.get("model", "gpt-4o")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
- # pylint: disable=no-else-return
if streaming:
awaited_wrapped = wrapped(*args, **kwargs)
span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
@@ -130,7 +116,6 @@ def complete(version, environment, application_name,
environment=environment,
application_name=application_name,
metrics=metrics,
- event_provider=event_provider,
start_time=start_time,
span=span,
capture_message_content=capture_message_content,
@@ -142,3 +127,48 @@ def complete(version, environment, application_name,
return response
return wrapper
+
+def embed(version, environment, application_name,
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for GenAI embedding function call
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the GenAI embedding function call.
+ """
+
+ server_address, server_port = set_server_address_and_port(instance, "models.github.ai", 443)
+ request_model = kwargs.get("model", "text-embedding-3-small")
+
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
+ response = wrapped(*args, **kwargs)
+
+ try:
+ response = process_embedding_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
index f45fb62d..321e5725 100644
--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/utils.py
@@ -3,104 +3,185 @@ Azure AI Inference OpenTelemetry instrumentation utility functions
"""
import time
-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
from opentelemetry.trace import Status, StatusCode
from openlit.__helpers import (
calculate_ttft,
response_as_dict,
calculate_tbt,
- extract_and_format_input,
get_chat_model_cost,
- create_metrics_attributes,
- otel_event,
- concatenate_all_contents
+ get_embed_model_cost,
+ common_span_attributes,
+ record_completion_metrics,
+ record_embedding_metrics,
)
from openlit.semcov import SemanticConvention
-def process_chunk(self, chunk):
+def format_content(messages):
+ """
+ Process a list of messages to extract content.
+ """
+
+ formatted_messages = []
+ for message in messages:
+ role = message.get("role", "user")
+ content = message.get("content", "")
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ f'{item["type"]}: {item["text"] if "text" in item else item.get("image_url", "")}'
+ if "type" in item else f'text: {item.get("text", "")}'
+ for item in content
+ )
+ formatted_messages.append(f'{role}: {content_str}')
+ else:
+ formatted_messages.append(f'{role}: {content}')
+
+ return '\n'.join(formatted_messages)
+
+def process_chunk(scope, chunk):
"""
Process a chunk of response data and update state.
"""
end_time = time.time()
# Record the timestamp for the current chunk
- self._timestamps.append(end_time)
+ scope._timestamps.append(end_time)
- if len(self._timestamps) == 1:
+ if len(scope._timestamps) == 1:
# Calculate time to first chunk
- self._ttft = calculate_ttft(self._timestamps, self._start_time)
+ scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)
chunked = response_as_dict(chunk)
# Collect message IDs and aggregated response from events
- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and
- 'content' in chunked.get('choices')[0].get('delta'))):
+ choices = chunked.get("choices", [])
+ if choices and "delta" in choices[0]:
+ delta = choices[0]["delta"]
+
+ # Handle content
+ content = delta.get("content")
+ if content:
+ scope._llmresponse += content
+
+ # Handle reasoning content (if present)
+ reasoning_content = delta.get("reasoning_content")
+ if reasoning_content:
+ if not hasattr(scope, "_reasoning_content"):
+ scope._reasoning_content = ""
+ scope._reasoning_content += reasoning_content
- if content := chunked.get('choices')[0].get('delta').get('content'):
- self._llmresponse += content
+ # Handle finish_reason (appears in final chunk)
+ finish_reason = chunked.get("choices")[0].get("finish_reason")
+ if finish_reason:
+ scope._finish_reason = finish_reason
+ scope._end_time = time.time()
- if chunked.get('choices')[0].get('finish_reason') is not None:
- self._finish_reason = chunked.get('choices')[0].get('finish_reason')
+ # Handle tool calls in streaming - optimized
+ delta_tools = delta.get("tool_calls")
+ if delta_tools:
+ scope._tools = scope._tools or []
- if chunked.get('usage') is not None:
- self._input_tokens = chunked.get('usage').get('prompt_tokens')
- self._response_id = chunked.get('id')
- self._response_model = chunked.get('model')
- self._output_tokens = chunked.get('usage').get('completion_tokens')
+ for tool in delta_tools:
+ idx = tool.get("index", 0)
+
+ # Extend list if needed
+ scope._tools.extend([{}] * (idx + 1 - len(scope._tools)))
+
+ if tool.get("id"): # New tool (id exists)
+ func = tool.get("function", {})
+ scope._tools[idx] = {
+ "id": tool["id"],
+ "function": {"name": func.get("name", ""), "arguments": func.get("arguments", "")},
+ "type": tool.get("type", "function")
+ }
+ elif scope._tools[idx] and "function" in tool: # Append args (id is None)
+ scope._tools[idx]["function"]["arguments"] += tool["function"].get("arguments", "")
+
+ # Handle usage information (typically only in final chunk)
+ if chunked.get("usage"):
+ scope._input_tokens = chunked.get("usage").get("prompt_tokens", 0)
+ scope._output_tokens = chunked.get("usage").get("completion_tokens", 0)
+ # Handle reasoning tokens if present (optional) - check nested structure
+ completion_details = chunked.get("usage", {}).get("completion_tokens_details", {})
+ if "reasoning_tokens" in completion_details:
+ scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)
+ elif "reasoning_tokens" in chunked.get("usage", {}):
+ scope._reasoning_tokens = chunked.get("usage").get("reasoning_tokens", 0)
def common_chat_logic(scope, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream):
+ capture_message_content, disable_metrics, version, is_stream):
"""
Process chat request and generate Telemetry
"""
- scope._end_time = time.time()
if len(scope._timestamps) > 1:
scope._tbt = calculate_tbt(scope._timestamps)
- formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))
- request_model = scope._kwargs.get('model', 'claude-3-opus-20240229')
+ prompt = format_content(scope._kwargs.get("messages", []))
+ request_model = scope._kwargs.get("model", "gpt-4o")
cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)
- # Set Span attributes (OTel Semconv)
- scope._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
- scope._span.set_attribute(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)
- scope._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE)
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL, request_model)
- scope._span.set_attribute(SemanticConvention.SERVER_PORT, scope._server_port)
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get('max_tokens', -1))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get('stop', []))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get('temperature', 1.0))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, scope._kwargs.get('top_k', 1.0))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get('top_p', 1.0))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,
- scope._kwargs.get('frequency_penalty', 0.0))
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,
- scope._kwargs.get('presence_penalty', 0.0))
- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])
+ # Common Span Attributes
+ common_span_attributes(scope,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,
+ scope._server_address, scope._server_port, request_model, scope._response_model,
+ environment, application_name, is_stream, scope._tbt, scope._ttft, version)
+
+ # Span Attributes for Request parameters
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, scope._kwargs.get("frequency_penalty", 0.0))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get("max_tokens", -1))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, scope._kwargs.get("presence_penalty", 0.0))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get("stop", []))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get("temperature", 1.0))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get("top_p", 1.0))
+
+ # Span Attributes for Response parameters
scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, scope._response_id)
- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, scope._response_model)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])
+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SERVICE_TIER, scope._response_service_tier)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT, scope._response_service_tier)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text" if isinstance(scope._llmresponse, str) else "json")
+
+ # Span Attributes for Cost and Tokens
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)
- scope._span.set_attribute(SemanticConvention.SERVER_ADDRESS, scope._server_address)
-
- scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,
- 'text' if isinstance(scope._llmresponse, str) else 'json')
-
- scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)
- scope._span.set_attribute(SERVICE_NAME, application_name)
- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, is_stream)
scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)
scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT, scope._tbt)
- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT, scope._ttft)
- scope._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, version)
- # To be removed one the change to log events (from span events) is complete
- prompt = concatenate_all_contents(formatted_messages)
+ # Span Attributes for Reasoning (if present)
+ if hasattr(scope, "_reasoning_tokens") and scope._reasoning_tokens > 0:
+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_REASONING_TOKENS, scope._reasoning_tokens)
+ # Update total token usage to include reasoning tokens
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
+ scope._input_tokens + scope._output_tokens + scope._reasoning_tokens)
+
+ # Span Attributes for Tools - optimized
+ if scope._tools:
+ tools = scope._tools if isinstance(scope._tools, list) else [scope._tools]
+
+ names, ids, args = zip(*[
+ (t.get("function", {}).get("name", ""),
+ str(t.get("id", "")),
+ str(t.get("function", {}).get("arguments", "")))
+ for t in tools if isinstance(t, dict) and t
+ ]) if tools else ([], [], [])
+
+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, ", ".join(filter(None, names)))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, ", ".join(filter(None, ids)))
+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, ", ".join(filter(None, args)))
+
+ # Span Attributes for Content
if capture_message_content:
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse)
+
+ # Add reasoning content if available
+ if hasattr(scope, "_reasoning_content") and scope._reasoning_content:
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_REASONING, scope._reasoning_content)
+
+ # To be removed once the change to span_attributes (from span events) is complete
scope._span.add_event(
name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
@@ -114,112 +195,143 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
},
)
- choice_event_body = {
- 'finish_reason': scope._finish_reason,
- 'index': 0,
- 'message': {
- **({'content': scope._llmresponse} if capture_message_content else {}),
- 'role': 'assistant'
- }
- }
-
- # Emit events
- for role in ['user', 'system', 'assistant', 'tool']:
- if formatted_messages.get(role, {}).get('content', ''):
- event = otel_event(
- name=getattr(SemanticConvention, f'GEN_AI_{role.upper()}_MESSAGE'),
- attributes={
- SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE
- },
- body = {
- # pylint: disable=line-too-long
- **({'content': formatted_messages.get(role, {}).get('content', '')} if capture_message_content else {}),
- 'role': formatted_messages.get(role, {}).get('role', []),
- **({
- 'tool_calls': {
- 'function': {
- # pylint: disable=line-too-long
- 'name': (scope._tool_calls[0].get('function', {}).get('name', '') if scope._tool_calls else ''),
- 'arguments': (scope._tool_calls[0].get('function', {}).get('arguments', '') if scope._tool_calls else '')
- },
- 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else ''),
- 'type': 'function'
- }
- } if role == 'assistant' else {}),
- **({
- 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else '')
- } if role == 'tool' else {})
- }
- )
- event_provider.emit(event)
-
- choice_event = otel_event(
- name=SemanticConvention.GEN_AI_CHOICE,
- attributes={
- SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE
- },
- body=choice_event_body
- )
- event_provider.emit(choice_event)
-
scope._span.set_status(Status(StatusCode.OK))
+ # Metrics
if not disable_metrics:
- metrics_attributes = create_metrics_attributes(
- service_name=application_name,
- deployment_environment=environment,
- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
- system=SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,
- request_model=request_model,
- server_address=scope._server_address,
- server_port=scope._server_port,
- response_model=scope._response_model,
- )
+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE, scope._server_address, scope._server_port,
+ request_model, scope._response_model, environment, application_name, scope._start_time, scope._end_time,
+ scope._input_tokens, scope._output_tokens, cost, scope._tbt, scope._ttft)
- metrics['genai_client_usage_tokens'].record(scope._input_tokens + scope._output_tokens, metrics_attributes)
- metrics['genai_client_operation_duration'].record(scope._end_time - scope._start_time, metrics_attributes)
- metrics['genai_server_tbt'].record(scope._tbt, metrics_attributes)
- metrics['genai_server_ttft'].record(scope._ttft, metrics_attributes)
- metrics['genai_requests'].add(1, metrics_attributes)
- metrics['genai_completion_tokens'].add(scope._output_tokens, metrics_attributes)
- metrics['genai_prompt_tokens'].add(scope._input_tokens, metrics_attributes)
- metrics['genai_cost'].record(cost, metrics_attributes)
-
-def process_streaming_chat_response(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content=False, disable_metrics=False, version=''):
+def process_streaming_chat_response(scope, pricing_info, environment, application_name, metrics,
+ capture_message_content=False, disable_metrics=False, version=""):
"""
- Process chat request and generate Telemetry
+ Process streaming chat request and generate Telemetry
"""
- common_chat_logic(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream=True)
+ common_chat_logic(scope, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version, is_stream=True)
def process_chat_response(response, request_model, pricing_info, server_port, server_address,
- environment, application_name, metrics, event_provider, start_time,
- span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
+ environment, application_name, metrics, start_time, span, capture_message_content=False,
+ disable_metrics=False, version="1.0.0", **kwargs):
"""
Process chat request and generate Telemetry
"""
- self = type('GenericScope', (), {})()
+ # Create scope object
+ scope = type("GenericScope", (), {})()
+ response_dict = response_as_dict(response)
+
+ scope._start_time = start_time
+ scope._end_time = time.time()
+ scope._span = span
+ scope._llmresponse = " ".join(
+ (choice.get("message", {}).get("content") or "")
+ for choice in response_dict.get("choices", [])
+ )
+ # Handle reasoning content from non-streaming response
+ reasoning_content = response_dict.get("choices", [{}])[0].get("message", {}).get("reasoning_content")
+ if reasoning_content:
+ scope._reasoning_content = reasoning_content
+
+ scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)
+ scope._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)
+ # Handle reasoning tokens if present (optional) - check nested structure
+ completion_details = response_dict.get("usage", {}).get("completion_tokens_details", {})
+ if "reasoning_tokens" in completion_details:
+ scope._reasoning_tokens = completion_details.get("reasoning_tokens", 0)
+ elif "reasoning_tokens" in response_dict.get("usage", {}):
+ scope._reasoning_tokens = response_dict.get("usage").get("reasoning_tokens", 0)
+ else:
+ scope._reasoning_tokens = 0
+ scope._response_id = response_dict.get("id")
+ scope._response_model = response_dict.get("model")
+ scope._finish_reason = str(response_dict.get("choices", [])[0].get("finish_reason", ""))
+ scope._response_service_tier = str(response_dict.get("system_fingerprint", ""))
+ scope._timestamps = []
+ scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0
+ scope._server_address, scope._server_port = server_address, server_port
+ scope._kwargs = kwargs
+
+ # Handle tool calls
+ if scope._kwargs.get("tools"):
+ scope._tools = response_dict.get("choices", [{}])[0].get("message", {}).get("tool_calls")
+ else:
+ scope._tools = None
+
+ common_chat_logic(scope, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version, is_stream=False)
+
+ return response
+
+def common_embedding_logic(scope, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version):
+ """
+ Process embedding request and generate Telemetry
+ """
+
+ request_model = scope._kwargs.get("model", "text-embedding-3-small")
+
+ cost = get_embed_model_cost(request_model, pricing_info, scope._input_tokens)
+
+ # Common Span Attributes
+ common_span_attributes(scope,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING, SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE,
+ scope._server_address, scope._server_port, request_model, scope._response_model,
+ environment, application_name, False, 0, scope._end_time - scope._start_time, version)
+
+ # Span Attributes for Request parameters
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS, [scope._kwargs.get("encoding_format", "float")])
+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER, scope._kwargs.get("user", ""))
+
+ # Span Attributes for Cost and Tokens
+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens)
+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
+
+ # Span Attributes for Content
+ if capture_message_content:
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, str(scope._kwargs.get("input", "")))
+
+ # To be removed once the change to span_attributes (from span events) is complete
+ scope._span.add_event(
+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={
+ SemanticConvention.GEN_AI_CONTENT_PROMPT: str(scope._kwargs.get("input", "")),
+ },
+ )
+
+ scope._span.set_status(Status(StatusCode.OK))
+
+ # Metrics
+ if not disable_metrics:
+ record_embedding_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING,
+ SemanticConvention.GEN_AI_SYSTEM_AZURE_AI_INFERENCE, scope._server_address, scope._server_port,
+ request_model, scope._response_model, environment, application_name, scope._start_time, scope._end_time,
+ scope._input_tokens, cost)
+
+def process_embedding_response(response, request_model, pricing_info, server_port, server_address,
+ environment, application_name, metrics, start_time, span, capture_message_content=False,
+ disable_metrics=False, version="1.0.0", **kwargs):
+ """
+ Process embedding request and generate Telemetry
+ """
+
+ # Create scope object
+ scope = type("GenericScope", (), {})()
response_dict = response_as_dict(response)
- # pylint: disable = no-member
- self._start_time = start_time
- self._end_time = time.time()
- self._span = span
- self._llmresponse = response_dict.get('choices', {})[0].get('message', '').get('content', '')
- self._input_tokens = response_dict.get('usage').get('prompt_tokens')
- self._output_tokens = response_dict.get('usage').get('completion_tokens')
- self._response_model = response_dict.get('model', '')
- self._finish_reason = response_dict.get('choices', {})[0].get('finish_reason', '')
- self._response_id = response_dict.get('id', '')
- self._timestamps = []
- self._ttft, self._tbt = self._end_time - self._start_time, 0
- self._server_address, self._server_port = server_address, server_port
- self._kwargs = kwargs
-
- common_chat_logic(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream=False)
+ scope._start_time = start_time
+ scope._end_time = time.time()
+ scope._span = span
+ scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)
+ scope._response_model = response_dict.get("model")
+ scope._server_address, scope._server_port = server_address, server_port
+ scope._kwargs = kwargs
+
+ common_embedding_logic(scope, pricing_info, environment, application_name, metrics,
+ capture_message_content, disable_metrics, version)
return response
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index d4b04b59..e0cc8b82 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -166,6 +166,7 @@ class SemanticConvention:
GEN_AI_CONTENT_COMPLETION_EVENT = "gen_ai.content.completion"
GEN_AI_CONTENT_COMPLETION = "gen_ai.completion"
GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt"
+ GEN_AI_CONTENT_REASONING = "gen_ai.content.reasoning"
# GenAI Rag
GEN_AI_RAG_MAX_SEGMENTS = "gen_ai.rag.max_segments"
diff --git a/sdk/python/tests/test_azure_ai_inference.py.hold b/sdk/python/tests/test_azure_ai_inference.py
similarity index 81%
rename from sdk/python/tests/test_azure_ai_inference.py.hold
rename to sdk/python/tests/test_azure_ai_inference.py
index d137a820..7cd80743 100644
--- a/sdk/python/tests/test_azure_ai_inference.py.hold
+++ b/sdk/python/tests/test_azure_ai_inference.py
@@ -24,13 +24,13 @@ from azure.core.credentials import AzureKeyCredential
# Initialize synchronous Azure AI Inference Chat client
sync_chat_client = ChatCompletionsClient(
- endpoint="https://models.inference.ai.azure.com",
+ endpoint="https://models.github.ai/inference",
credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),
)
# Initialize synchronous Azure AI Inference Chat client
sync_embed_client = EmbeddingsClient(
- endpoint="https://models.inference.ai.azure.com",
+ endpoint="https://models.github.ai/inference",
credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),
)
@@ -39,7 +39,7 @@ openlit.init(environment="openlit-testing", application_name="openlit-python-tes
def test_sync_chat_completions():
"""
- Tests synchronous chat completions with the "Phi-3-mini-4k-instruct" model.
+ Tests synchronous chat completions with the "microsoft/Phi-4-mini-reasoning" model.
Raises:
AssertionError: If the chat completion response object is not as expected.
@@ -50,7 +50,7 @@ def test_sync_chat_completions():
SystemMessage(content="You are a helpful assistant."),
UserMessage(content="sync non-streaming"),
],
- model="Phi-3-mini-4k-instruct",
+ model="microsoft/Phi-4-mini-reasoning",
temperature=1.0,
max_tokens=1000,
top_p=1.0
@@ -59,7 +59,7 @@ def test_sync_chat_completions():
def test_sync_embeddings():
"""
- Tests synchronous embedding creation with the "cohere-embed-v3-english" model.
+ Tests synchronous embedding creation with the "openai/text-embedding-3-small" model.
Raises:
AssertionError: If the embedding response object is not as expected.
@@ -67,25 +67,25 @@ def test_sync_embeddings():
response = sync_embed_client.embed(
input=["LLM Observability", "Monitor GPUs"],
- model="cohere-embed-v3-english"
+ model="openai/text-embedding-3-small"
)
assert response.data[0]["object"] == 'embedding'
from azure.ai.inference.aio import ChatCompletionsClient, EmbeddingsClient
async_chat_client = ChatCompletionsClient(
- endpoint="https://models.inference.ai.azure.com",
+ endpoint="https://models.github.ai/inference",
credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),
)
async_embed_client = EmbeddingsClient(
- endpoint="https://models.inference.ai.azure.com",
+ endpoint="https://models.github.ai/inference",
credential=AzureKeyCredential(os.getenv("AZURE_AI_INFERENCE_API_TOKEN")),
)
@pytest.mark.asyncio
async def test_async_chat_completions():
"""
- Tests synchronous chat completions with the "Phi-3-mini-4k-instruct" model.
+ Tests synchronous chat completions with the "microsoft/Phi-4-mini-reasoning" model.
Raises:
AssertionError: If the chat completion response object is not as expected.
@@ -96,7 +96,7 @@ async def test_async_chat_completions():
SystemMessage(content="You are a helpful assistant."),
UserMessage(content="sync non-streaming"),
],
- model="Phi-3-mini-4k-instruct",
+ model="microsoft/Phi-4-mini-reasoning",
temperature=1.0,
max_tokens=1000,
top_p=1.0
@@ -106,7 +106,7 @@ async def test_async_chat_completions():
@pytest.mark.asyncio
async def test_async_embeddings():
"""
- Tests synchronous embedding creation with the "cohere-embed-v3-english" model.
+ Tests synchronous embedding creation with the "openai/text-embedding-3-small" model.
Raises:
AssertionError: If the embedding response object is not as expected.
@@ -114,6 +114,6 @@ async def test_async_embeddings():
response = await async_embed_client.embed(
input=["LLM Observability", "Monitor GPUs"],
- model="cohere-embed-v3-english"
+ model="openai/text-embedding-3-small"
)
assert response.data[0]["object"] == 'embedding'