Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Update Mistral and Ollama OpenTelemetry Instrumentation and align with Semantic Conventions (#628)

Patcherdb8d31da9ffb
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index dde0383f..eea7ee41 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.33.9"
+version = "1.33.10"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index 01a12c6c..5a7de73a 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -266,6 +266,11 @@ def set_server_address_and_port(client_instance: Any,
config = getattr(client_instance, "_config", None)
base_url = getattr(config, "endpoint", None)
+ if not base_url:
+ # Attempt to get server_url from instance.sdk_configuration.server_url
+ config = getattr(client_instance, "sdk_configuration", None)
+ base_url = getattr(config, "server_url", None)
+
if base_url:
if isinstance(base_url, str):
url = urlparse(base_url)
diff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py
index 54440aec..2a375d81 100644
--- a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py
+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py
@@ -60,8 +60,7 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,
start_time = time.time()
response = await wrapped(*args, **kwargs)
end_time = time.time()
- for key, value in instance.__dict__.items():
- print(f'{key}: {value}')
+
try:
# Calculate cost of the operation
cost = get_audio_model_cost(request_model,
diff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py
index 7b3d9cec..4dc6821a 100644
--- a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py
+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py
@@ -61,8 +61,7 @@ def generate(gen_ai_endpoint, version, environment, application_name,
start_time = time.time()
response = wrapped(*args, **kwargs)
end_time = time.time()
- for key, value in instance.__dict__.items():
- print(f'{key}: {value}')
+
try:
# Calculate cost of the operation
cost = get_audio_model_cost(request_model,
diff --git a/sdk/python/src/openlit/instrumentation/mistral/__init__.py b/sdk/python/src/openlit/instrumentation/mistral/__init__.py
index 123f6e4b..7bc0074d 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/__init__.py
@@ -31,7 +31,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.chat",
"Chat.complete",
- chat("mistral.chat", version, environment, application_name,
+ chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -39,7 +39,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.chat",
"Chat.stream",
- chat_stream("mistral.chat", version, environment, application_name,
+ chat_stream(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -47,7 +47,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.embeddings",
"Embeddings.create",
- embeddings("mistral.embeddings", version, environment, application_name,
+ embeddings(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -55,7 +55,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.chat",
"Chat.complete_async",
- async_chat("mistral.chat", version, environment, application_name,
+ async_chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -63,7 +63,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.chat",
"Chat.stream_async",
- async_chat_stream("mistral.chat", version, environment, application_name,
+ async_chat_stream(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -71,7 +71,7 @@ class MistralInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"mistralai.embeddings",
"Embeddings.create_async",
- async_embeddings("mistral.embeddings", version, environment, application_name,
+ async_embeddings(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
index 8eb8f635..e92bc284 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
@@ -1,29 +1,37 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
"""
Module for monitoring Mistral API calls.
"""
import logging
+import time
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.__helpers import (
+ get_chat_model_cost,
+ get_embed_model_cost,
+ handle_exception,
+ response_as_dict,
+ calculate_ttft,
+ calculate_tbt,
+ create_metrics_attributes,
+ set_server_address_and_port
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def async_chat(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def async_chat(version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for chat to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
@@ -47,71 +55,95 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,
The response from the original 'chat' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- # Handling exception ensure observability without disrupting operation
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-small-latest')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
response = await wrapped(*args, **kwargs)
+ end_time = time.time()
+
+ response_dict = response_as_dict(response)
try:
# Format 'messages' into a single string
- message_prompt = kwargs.get('messages', "")
+ message_prompt = kwargs.get('messages', '')
formatted_messages = []
for message in message_prompt:
- role = message["role"]
- content = message["content"]
+ role = message['role']
+ content = message['content']
if isinstance(content, list):
content_str = ", ".join(
- # pylint: disable=line-too-long
- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
- if 'type' in item else f"text: {item['text']}"
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
for item in content
)
- formatted_messages.append(f"{role}: {content_str}")
+ formatted_messages.append(f'{role}: {content_str}')
else:
- formatted_messages.append(f"{role}: {content}")
- prompt = " ".join(formatted_messages)
+ formatted_messages.append(f'{role}: {content}')
+ prompt = '\n'.join(formatted_messages)
+
+ input_tokens = response_dict.get('usage').get('prompt_tokens')
+ output_tokens = response_dict.get('usage').get('completion_tokens')
# Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
- pricing_info, response.usage.prompt_tokens,
- response.usage.completion_tokens)
+ cost = get_chat_model_cost(request_model,
+ pricing_info, input_tokens,
+ output_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ # Set base span attribues (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get('seed', ''))
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get('frequency_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get('max_tokens', -1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get('presence_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
+ kwargs.get('stop', []))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get('temperature', 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get('top_p', 1.0))
span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response.id)
+ response_dict.get('id'))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+
+ # Set base span attribues (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "mistral-small-latest"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 0.7))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
- kwargs.get("random_seed", ""))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response.choices[0].finish_reason])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- response.usage.prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- response.usage.completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- response.usage.total_tokens)
+ input_tokens + output_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ end_time - start_time)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
@@ -119,64 +151,80 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,
SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- # pylint: disable=line-too-long
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content if response.choices[0].message.content else "",
- },
- )
+
+ for i in range(kwargs.get('n',1)):
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [response_dict.get('choices')[i].get('finish_reason')])
+ if trace_content:
+ span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ # pylint: disable=line-too-long
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),
+ },
+ )
+ if kwargs.get('tools'):
+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,
+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))
+
+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
+ elif response_dict.get('choices')[i].get('message').get('content') is not None:
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "mistral-small-latest")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
- metrics["genai_completion_tokens"].add(
- response.usage.completion_tokens, attributes
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
+ )
+
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens + output_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_server_ttft'].record(
+ end_time - start_time, attributes
)
- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
- metrics["genai_cost"].record(cost)
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_completion_tokens'].add(output_tokens, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
return wrapper
-def async_chat_stream(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def async_chat_stream(version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for chat_stream to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
@@ -200,149 +248,260 @@ def async_chat_stream(gen_ai_endpoint, version, environment, application_name,
The response from the original 'chat_stream' method.
"""
- async def stream_generator():
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ class TracedAsyncStream:
+ """
+ Wrapper for streaming responses to collect metrics and trace data.
+ Wraps the 'mistral.syncStream' response to collect message IDs and aggregated response.
+
+ This class implements the '__aiter__' and '__anext__' methods that
+ handle asynchronous streaming responses.
+
+ This class also implements '__aenter__' and '__aexit__' methods that
+ handle asynchronous context management protocol.
+ """
+ def __init__(
+ self,
+ wrapped,
+ span,
+ kwargs,
+ server_address,
+ server_port,
+ **args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
# Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- async for event in await wrapped(*args, **kwargs):
- response_id = event.data.id
- llmresponse += event.data.choices[0].delta.content
- if event.data.usage is not None:
- prompt_tokens = event.data.usage.prompt_tokens
- completion_tokens = event.data.usage.completion_tokens
- total_tokens = event.data.usage.total_tokens
- finish_reason = event.data.choices[0].finish_reason
- yield event
-
- # Handling exception ensure observability without disrupting operation
+ self._llmresponse = ''
+ self._response_id = ''
+ self._response_model = ''
+ self._finish_reason = ''
+ self._input_tokens = ''
+ self._output_tokens = ''
+
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.time()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ async def __aenter__(self):
+ await self.__wrapped__.__aenter__()
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+
+ def __aiter__(self):
+ return self
+
+ async def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(await self.__wrapped__, name)
+
+ async def __anext__(self):
try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get('messages', "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
- if 'type' in item else f"text: {item['text']}"
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
+ chunk = await self.__wrapped__.__anext__()
+ end_time = time.time()
+ # Record the timestamp for the current chunk
+ self._timestamps.append(end_time)
+
+ if len(self._timestamps) == 1:
+ # Calculate time to first chunk
+ self._ttft = calculate_ttft(self._timestamps, self._start_time)
+
+ chunked = response_as_dict(chunk)
+
+ self._llmresponse += chunked.get('data').get('choices')[0].get('delta').get('content')
+ if chunked.get('data').get('usage') is not None:
+ self._response_id = chunked.get('data').get('id')
+ self._response_model = chunked.get('data').get('model')
+ self._input_tokens = chunked.get('data').get('usage').get('prompt_tokens')
+ self._output_tokens = chunked.get('data').get('usage').get('completion_tokens')
+ self._finish_reason = chunked.get('data').get('choices')[0].get('finish_reason')
+
+ return chunk
+ except StopAsyncIteration:
+ # Handling exception ensure observability without disrupting operation
+ try:
+ self._end_time = time.time()
+ if len(self._timestamps) > 1:
+ self._tbt = calculate_tbt(self._timestamps)
+
+ # Format 'messages' into a single string
+ message_prompt = self._kwargs.get('messages', '')
+ formatted_messages = []
+ for message in message_prompt:
+ role = message['role']
+ content = message['content']
+
+ if isinstance(content, list):
+ content_str_list = []
+ for item in content:
+ if item['type'] == 'text':
+ content_str_list.append(f'text: {item["text"]}')
+ elif (item['type'] == 'image_url' and
+ not item['image_url']['url'].startswith('data:')):
+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')
+ content_str = ", ".join(content_str_list)
+ formatted_messages.append(f'{role}: {content_str}')
+ else:
+ formatted_messages.append(f'{role}: {content}')
+ prompt = '\n'.join(formatted_messages)
+
+ request_model = self._kwargs.get('model', 'mistral-small-latest')
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, self._input_tokens,
+ self._output_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ self._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ self._kwargs.get('seed', ''))
+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,
+ self._server_port)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ self._kwargs.get('frequency_penalty', 0.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ self._kwargs.get('max_tokens', -1))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ self._kwargs.get('presence_penalty', 0.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
+ self._kwargs.get('stop_sequences', []))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ self._kwargs.get('temperature', 0.3))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ self._kwargs.get('k', 1.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ self._kwargs.get('p', 1.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [self._finish_reason])
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ self._response_id)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ self._response_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ self._input_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ self._server_address)
+
+ if isinstance(self._llmresponse, str):
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
else:
- formatted_messages.append(f"{role}: {content}")
- prompt = " ".join(formatted_messages)
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response_id)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "mistral-small-latest"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 0.7))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
- kwargs.get("random_seed", ""))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [finish_reason])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
+
+ # Set Span attributes (Extra)
+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
+ environment)
+ self._span.set_attribute(SERVICE_NAME,
+ application_name)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ self._input_tokens + self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
+ self._tbt)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ self._ttft)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+ if trace_content:
+ self._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
+ },
+ )
+ self._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
+ },
+ )
+ self._span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=self._server_address,
+ server_port=self._server_port,
+ response_model=self._response_model,
+ )
+
+ metrics['genai_client_usage_tokens'].record(
+ self._input_tokens + self._output_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ self._end_time - self._start_time, attributes
+ )
+ metrics['genai_server_tbt'].record(
+ self._tbt, attributes
+ )
+ metrics['genai_server_ttft'].record(
+ self._ttft, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_completion_tokens'].add(self._output_tokens, attributes)
+ metrics['genai_prompt_tokens'].add(self._input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(self._span, e)
+ logger.error('Error in trace creation: %s', e)
+ finally:
+ self._span.end()
+ raise
+
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-small-latest')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "mistral-small-latest")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
+ awaited_wrapped = await wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
return wrapper
-def async_embeddings(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def async_embeddings(version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for embeddings to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
A function that wraps the embeddings method to add telemetry.
"""
- async def wrapper(wrapped, instance, args, kwargs):
+ def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the 'embeddings' API call to add telemetry.
@@ -359,78 +518,92 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,
The response from the original 'embeddings' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- response = await wrapped(*args, **kwargs)
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-embed')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
+ response = wrapped(*args, **kwargs)
+ end_time = time.time()
+
+ response_dict = response_as_dict(response)
try:
- # Get prompt from kwargs and store as a single string
- prompt = ', '.join(kwargs.get('inputs', []))
+ input_tokens = response_dict.get('usage').get('prompt_tokens')
# Calculate cost of the operation
- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
- pricing_info, response.usage.prompt_tokens)
+ cost = get_embed_model_cost(request_model,
+ pricing_info, input_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ # Set Span attributes (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,
+ [kwargs.get('encoding_format', 'float')])
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+
+ # Set Span attributes (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get('model', "mistral-embed"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,
- kwargs.get("encoding_format", "float"))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response.id)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- response.usage.prompt_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- response.usage.total_tokens)
+ input_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('inputs', '')),
},
)
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get('model', "mistral-embed")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
+ )
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
diff --git a/sdk/python/src/openlit/instrumentation/mistral/mistral.py b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
index 5b2b6583..3dd363f6 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/mistral.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
@@ -1,29 +1,37 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
"""
Module for monitoring Mistral API calls.
"""
import logging
+import time
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.__helpers import (
+ get_chat_model_cost,
+ get_embed_model_cost,
+ handle_exception,
+ response_as_dict,
+ calculate_ttft,
+ calculate_tbt,
+ create_metrics_attributes,
+ set_server_address_and_port
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def chat(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def chat(version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for chat to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
@@ -47,70 +55,95 @@ def chat(gen_ai_endpoint, version, environment, application_name,
The response from the original 'chat' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-small-latest')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
response = wrapped(*args, **kwargs)
+ end_time = time.time()
+
+ response_dict = response_as_dict(response)
try:
# Format 'messages' into a single string
- message_prompt = kwargs.get('messages', "")
+ message_prompt = kwargs.get('messages', '')
formatted_messages = []
for message in message_prompt:
- role = message["role"]
- content = message["content"]
+ role = message['role']
+ content = message['content']
if isinstance(content, list):
content_str = ", ".join(
- # pylint: disable=line-too-long
- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
- if 'type' in item else f"text: {item['text']}"
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
for item in content
)
- formatted_messages.append(f"{role}: {content_str}")
+ formatted_messages.append(f'{role}: {content_str}')
else:
- formatted_messages.append(f"{role}: {content}")
- prompt = " ".join(formatted_messages)
+ formatted_messages.append(f'{role}: {content}')
+ prompt = '\n'.join(formatted_messages)
+
+ input_tokens = response_dict.get('usage').get('prompt_tokens')
+ output_tokens = response_dict.get('usage').get('completion_tokens')
# Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
- pricing_info, response.usage.prompt_tokens,
- response.usage.completion_tokens)
+ cost = get_chat_model_cost(request_model,
+ pricing_info, input_tokens,
+ output_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ # Set base span attribues (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get('seed', ''))
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get('frequency_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get('max_tokens', -1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get('presence_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
+ kwargs.get('stop', []))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get('temperature', 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get('top_p', 1.0))
span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response.id)
+ response_dict.get('id'))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+
+ # Set base span attribues (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "mistral-small-latest"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 0.7))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
- kwargs.get("random_seed", ""))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response.choices[0].finish_reason])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- response.usage.prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- response.usage.completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- response.usage.total_tokens)
+ input_tokens + output_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ end_time - start_time)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
@@ -118,64 +151,80 @@ def chat(gen_ai_endpoint, version, environment, application_name,
SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- # pylint: disable=line-too-long
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content if response.choices[0].message.content else "",
- },
- )
+
+ for i in range(kwargs.get('n',1)):
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [response_dict.get('choices')[i].get('finish_reason')])
+ if trace_content:
+ span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ # pylint: disable=line-too-long
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),
+ },
+ )
+ if kwargs.get('tools'):
+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,
+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))
+
+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
+ elif response_dict.get('choices')[i].get('message').get('content') is not None:
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "mistral-small-latest")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
- metrics["genai_completion_tokens"].add(
- response.usage.completion_tokens, attributes
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
)
- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens + output_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_server_ttft'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_completion_tokens'].add(output_tokens, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
return wrapper
-def chat_stream(gen_ai_endpoint, version, environment, application_name,
+def chat_stream(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for chat_stream to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
@@ -199,142 +248,253 @@ def chat_stream(gen_ai_endpoint, version, environment, application_name,
The response from the original 'chat_stream' method.
"""
- def stream_generator():
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ class TracedSyncStream:
+ """
+ Wrapper for streaming responses to collect metrics and trace data.
+ Wraps the 'mistral.syncStream' response to collect message IDs and aggregated response.
+
+ This class implements the '__aiter__' and '__anext__' methods that
+ handle asynchronous streaming responses.
+
+ This class also implements '__aenter__' and '__aexit__' methods that
+ handle asynchronous context management protocol.
+ """
+ def __init__(
+ self,
+ wrapped,
+ span,
+ kwargs,
+ server_address,
+ server_port,
+ **args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
# Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- for event in wrapped(*args, **kwargs):
- response_id = event.data.id
- llmresponse += event.data.choices[0].delta.content
- if event.data.usage is not None:
- prompt_tokens = event.data.usage.prompt_tokens
- completion_tokens = event.data.usage.completion_tokens
- total_tokens = event.data.usage.total_tokens
- finish_reason = event.data.choices[0].finish_reason
- yield event
-
- # Handling exception ensure observability without disrupting operation
+ self._llmresponse = ''
+ self._response_id = ''
+ self._response_model = ''
+ self._finish_reason = ''
+ self._input_tokens = ''
+ self._output_tokens = ''
+
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.time()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ def __enter__(self):
+ self.__wrapped__.__enter__()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+
+ def __iter__(self):
+ return self
+
+ def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(self.__wrapped__, name)
+
+ def __next__(self):
try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get('messages', "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
- if 'type' in item else f"text: {item['text']}"
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
+ chunk = self.__wrapped__.__next__()
+ end_time = time.time()
+ # Record the timestamp for the current chunk
+ self._timestamps.append(end_time)
+
+ if len(self._timestamps) == 1:
+ # Calculate time to first chunk
+ self._ttft = calculate_ttft(self._timestamps, self._start_time)
+
+ chunked = response_as_dict(chunk)
+
+ self._llmresponse += chunked.get('data').get('choices')[0].get('delta').get('content')
+ if chunked.get('data').get('usage') is not None:
+ self._response_id = chunked.get('data').get('id')
+ self._response_model = chunked.get('data').get('model')
+ self._input_tokens = chunked.get('data').get('usage').get('prompt_tokens')
+ self._output_tokens = chunked.get('data').get('usage').get('completion_tokens')
+ self._finish_reason = chunked.get('data').get('choices')[0].get('finish_reason')
+
+ return chunk
+ except StopIteration:
+ # Handling exception ensure observability without disrupting operation
+ try:
+ self._end_time = time.time()
+ if len(self._timestamps) > 1:
+ self._tbt = calculate_tbt(self._timestamps)
+
+ # Format 'messages' into a single string
+ message_prompt = self._kwargs.get('messages', '')
+ formatted_messages = []
+ for message in message_prompt:
+ role = message['role']
+ content = message['content']
+
+ if isinstance(content, list):
+ content_str_list = []
+ for item in content:
+ if item['type'] == 'text':
+ content_str_list.append(f'text: {item["text"]}')
+ elif (item['type'] == 'image_url' and
+ not item['image_url']['url'].startswith('data:')):
+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')
+ content_str = ", ".join(content_str_list)
+ formatted_messages.append(f'{role}: {content_str}')
+ else:
+ formatted_messages.append(f'{role}: {content}')
+ prompt = '\n'.join(formatted_messages)
+
+ request_model = self._kwargs.get('model', 'mistral-small-latest')
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, self._input_tokens,
+ self._output_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ self._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ self._kwargs.get('seed', ''))
+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,
+ self._server_port)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ self._kwargs.get('frequency_penalty', 0.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ self._kwargs.get('max_tokens', -1))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ self._kwargs.get('presence_penalty', 0.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
+ self._kwargs.get('stop_sequences', []))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ self._kwargs.get('temperature', 0.3))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ self._kwargs.get('k', 1.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ self._kwargs.get('p', 1.0))
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [self._finish_reason])
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ self._response_id)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ self._response_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ self._input_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ self._server_address)
+
+ if isinstance(self._llmresponse, str):
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
else:
- formatted_messages.append(f"{role}: {content}")
- prompt = " ".join(formatted_messages)
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response_id)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "mistral-small-latest"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 0.7))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
- kwargs.get("random_seed", ""))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [finish_reason])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
+
+ # Set Span attributes (Extra)
+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
+ environment)
+ self._span.set_attribute(SERVICE_NAME,
+ application_name)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ self._input_tokens + self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
+ self._tbt)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ self._ttft)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+ if trace_content:
+ self._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
+ },
+ )
+ self._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
+ },
+ )
+ self._span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=self._server_address,
+ server_port=self._server_port,
+ response_model=self._response_model,
+ )
+
+ metrics['genai_client_usage_tokens'].record(
+ self._input_tokens + self._output_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ self._end_time - self._start_time, attributes
+ )
+ metrics['genai_server_tbt'].record(
+ self._tbt, attributes
+ )
+ metrics['genai_server_ttft'].record(
+ self._ttft, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_completion_tokens'].add(self._output_tokens, attributes)
+ metrics['genai_prompt_tokens'].add(self._input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(self._span, e)
+ logger.error('Error in trace creation: %s', e)
+ finally:
+ self._span.end()
+ raise
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "mistral-small-latest")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-small-latest')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+
+ awaited_wrapped = wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
return wrapper
-def embeddings(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def embeddings(version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for embeddings to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the OpenAI API.
+ application_name: Name of the application using the Mistral API.
tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of OpenAI usage.
+ pricing_info: Information used for calculating the cost of Mistral usage.
trace_content: Flag indicating whether to trace the actual content.
Returns:
@@ -358,78 +518,92 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,
The response from the original 'embeddings' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)
+ request_model = kwargs.get('model', 'mistral-embed')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'
+
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
response = wrapped(*args, **kwargs)
+ end_time = time.time()
+ response_dict = response_as_dict(response)
try:
- # Get prompt from kwargs and store as a single string
- prompt = ', '.join(kwargs.get('inputs', []))
+ input_tokens = response_dict.get('usage').get('prompt_tokens')
# Calculate cost of the operation
- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
- pricing_info, response.usage.prompt_tokens)
+ cost = get_embed_model_cost(request_model,
+ pricing_info, input_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ # Set Span attributes (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,
+ [kwargs.get('encoding_format', 'float')])
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+
+ # Set Span attributes (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get('model', "mistral-embed"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,
- kwargs.get("encoding_format", "float"))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response.id)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- response.usage.prompt_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- response.usage.total_tokens)
+ input_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('inputs', '')),
},
)
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get('model', "mistral-embed")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
+ )
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
diff --git a/sdk/python/src/openlit/instrumentation/ollama/__init__.py b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
index b32bd094..d0f68237 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
@@ -7,10 +7,10 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from wrapt import wrap_function_wrapper
from openlit.instrumentation.ollama.ollama import (
- chat, embeddings, generate
+ chat, embeddings
)
from openlit.instrumentation.ollama.async_ollama import (
- async_chat, async_embeddings, async_generate
+ async_chat, async_embeddings
)
_instruments = ("ollama >= 0.2.0",)
@@ -37,13 +37,13 @@ class OllamaInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"ollama",
"chat",
- chat("ollama.chat", version, environment, application_name,
+ chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
wrap_function_wrapper(
"ollama",
"Client.chat",
- chat("ollama.chat", version, environment, application_name,
+ chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -51,27 +51,13 @@ class OllamaInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"ollama",
"embeddings",
- embeddings("ollama.embeddings", version, environment, application_name,
+ embeddings(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
wrap_function_wrapper(
"ollama",
"Client.embeddings",
- embeddings("ollama.embeddings", version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics),
- )
-
- # sync generate
- wrap_function_wrapper(
- "ollama",
- "generate",
- generate("ollama.generate", version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics),
- )
- wrap_function_wrapper(
- "ollama",
- "Client.generate",
- generate("ollama.generate", version, environment, application_name,
+ embeddings(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -79,7 +65,7 @@ class OllamaInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"ollama",
"AsyncClient.chat",
- async_chat("ollama.chat", version, environment, application_name,
+ async_chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
@@ -87,15 +73,7 @@ class OllamaInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"ollama",
"AsyncClient.embeddings",
- async_embeddings("ollama.embeddings", version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics),
- )
-
- # aync generate
- wrap_function_wrapper(
- "ollama",
- "AsyncClient.generate",
- async_generate("ollama.generate", version, environment, application_name,
+ async_embeddings(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics),
)
diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
index 493284f7..18c9c1d1 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
@@ -1,28 +1,33 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
"""
Module for monitoring Ollama API calls.
"""
import logging
+import time
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
from openlit.__helpers import (
+ get_chat_model_cost,
+ get_embed_model_cost,
handle_exception,
+ response_as_dict,
general_tokens,
- get_chat_model_cost,
- get_embed_model_cost)
+ calculate_ttft,
+ calculate_tbt,
+ create_metrics_attributes,
+ set_server_address_and_port
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def async_chat(gen_ai_endpoint, version, environment, application_name,
+def async_chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
- Generates a telemetry wrapper for chat to collect metrics.
+ Generates a telemetry wrapper for chat completions to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
application_name: Name of the application using the Ollama API.
@@ -31,464 +36,410 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,
trace_content: Flag indicating whether to trace the actual content.
Returns:
- A function that wraps the chat method to add telemetry.
+ A function that wraps the chat completions method to add telemetry.
"""
- async def wrapper(wrapped, instance, args, kwargs):
+ class TracedAsyncStream:
"""
- Wraps the 'chat' API call to add telemetry.
-
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
+ Wrapper for streaming responses to collect metrics and trace data.
+ Wraps the response to collect message IDs and aggregated response.
- Args:
- wrapped: The original 'chat' method to be wrapped.
- instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'chat' method.
- kwargs: Keyword arguments for the 'chat' method.
+ This class implements the '__aiter__' and '__anext__' methods that
+ handle asynchronous streaming responses.
- Returns:
- The response from the original 'chat' method.
+ This class also implements '__aenter__' and '__aexit__' methods that
+ handle asynchronous context management protocol.
"""
-
- # Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
-
- # pylint: disable=no-else-return
- if streaming:
- # Special handling for streaming response to accommodate the nature of data flow
- async def stream_generator():
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- # Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- async for chunk in await wrapped(*args, **kwargs):
- # Collect aggregated response from events
- content = chunk['message']['content']
- llmresponse += content
-
- if chunk['done'] is True:
- completion_tokens = chunk["eval_count"]
-
- yield chunk
-
- # Handling exception ensure observability without disrupting operation
- try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- prompt_tokens = general_tokens(prompt)
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
-
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
-
- # Handling for non-streaming responses
- else:
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- response = await wrapped(*args, **kwargs)
-
+ def __init__(
+ self,
+ wrapped,
+ span,
+ kwargs,
+ server_address,
+ server_port,
+ **args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
+ # Placeholder for aggregating streaming response
+ self._llmresponse = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._input_tokens = 0
+ self._output_tokens = 0
+
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.time()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ async def __aenter__(self):
+ await self.__wrapped__.__aenter__()
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+
+ def __aiter__(self):
+ return self
+
+ async def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(await self.__wrapped__, name)
+
+ async def __anext__(self):
+ try:
+ chunk = await self.__wrapped__.__anext__()
+ end_time = time.time()
+ # Record the timestamp for the current chunk
+ self._timestamps.append(end_time)
+
+ if len(self._timestamps) == 1:
+ # Calculate time to first chunk
+ self._ttft = calculate_ttft(self._timestamps, self._start_time)
+
+ chunked = response_as_dict(chunk)
+ self._llmresponse += chunked.get('message').get('content')
+
+ if chunked.get('eval_count'):
+ self._input_tokens = chunked.get('prompt_eval_count')
+ self._output_tokens = chunked.get('eval_count')
+ self._response_model = chunked.get('model')
+ self._finish_reason = chunked.get('done_reason')
+ return chunk
+ except StopAsyncIteration:
+ # Handling exception ensure observability without disrupting operation
try:
+ self._end_time = time.time()
+ if len(self._timestamps) > 1:
+ self._tbt = calculate_tbt(self._timestamps)
+
# Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
+ message_prompt = self._kwargs.get("messages", "")
formatted_messages = []
for message in message_prompt:
role = message["role"]
content = message["content"]
if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
+ content_str_list = []
+ for item in content:
+ if item["type"] == "text":
+ content_str_list.append(f'text: {item["text"]}')
+ elif (item["type"] == "image_url" and
+ not item["image_url"]["url"].startswith("data:")):
+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')
+ content_str = ", ".join(content_str_list)
formatted_messages.append(f"{role}: {content_str}")
else:
formatted_messages.append(f"{role}: {content}")
prompt = "\n".join(formatted_messages)
- # Set base span attribues
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
+ request_model = self._kwargs.get("model", "gpt-4o")
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, self._input_tokens,
+ self._output_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,
+ self._server_port)
+
+ # List of attributes and their config keys
+ attributes = [
+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),
+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),
+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),
+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),
+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),
+ ]
+
+ # Safely get the options dictionary from kwargs
+ options = self._kwargs.get('options', {})
+
+ # Set each attribute if the corresponding value exists and is not None
+ for attribute, key in attributes:
+ # Use dictionary `get` to retrieve values from the options dictionary
+ value = options.get(key)
+ if value is not None:
+ self._span.set_attribute(attribute, value)
+
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [self._finish_reason])
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ self._response_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ self._input_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ self._server_address)
+ if isinstance(self._llmresponse, str):
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ "text")
+ else:
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ "json")
+
+ # Set Span attributes (Extra)
+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
- span.set_attribute(SERVICE_NAME,
+ self._span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- False)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ self._input_tokens + self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
+ self._tbt)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ self._ttft)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
- span.add_event(
+ self._span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
- span.add_event(
+ self._span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['message']['content'],
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
},
)
-
- prompt_tokens = general_tokens(prompt)
- completion_tokens = response["eval_count"]
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response["done_reason"]])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
-
- span.set_status(Status(StatusCode.OK))
+ self._span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=self._server_address,
+ server_port=self._server_port,
+ response_model=self._response_model,
+ )
+ metrics["genai_client_usage_tokens"].record(
+ self._input_tokens + self._output_tokens, attributes
+ )
+ metrics["genai_client_operation_duration"].record(
+ self._end_time - self._start_time, attributes
+ )
+ metrics["genai_server_tbt"].record(
+ self._tbt, attributes
+ )
+ metrics["genai_server_ttft"].record(
+ self._ttft, attributes
+ )
metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)
metrics["genai_cost"].record(cost, attributes)
- # Return original response
- return response
-
except Exception as e:
- handle_exception(span, e)
+ handle_exception(self._span, e)
logger.error("Error in trace creation: %s", e)
-
- # Return original response
- return response
-
- return wrapper
-
-def async_generate(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
- """
- Generates a telemetry wrapper for generate to collect metrics.
-
- Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
- version: Version of the monitoring package.
- environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Ollama API.
- tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of Ollama usage.
- trace_content: Flag indicating whether to trace the actual content.
-
- Returns:
- A function that wraps the generate method to add telemetry.
- """
+ finally:
+ self._span.end()
+ raise
async def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'generate' API call to add telemetry.
-
+ Wraps the 'chat.completions' API call to add telemetry.
+
This collects metrics such as execution time, cost, and token usage, and handles errors
gracefully, adding details to the trace for observability.
Args:
- wrapped: The original 'generate' method to be wrapped.
+ wrapped: The original 'chat.completions' method to be wrapped.
instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'generate' method.
- kwargs: Keyword arguments for the 'generate' method.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
Returns:
- The response from the original 'generate' method.
+ The response from the original 'chat.completions' method.
"""
# Check if streaming is enabled for the API call
streaming = kwargs.get("stream", False)
+ server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)
+ request_model = kwargs.get("model", "gpt-4o")
+
+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
# pylint: disable=no-else-return
if streaming:
# Special handling for streaming response to accommodate the nature of data flow
- async def stream_generator():
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- # Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- async for chunk in await wrapped(*args, **kwargs):
- # Collect aggregated response from events
- content = chunk['response']
- llmresponse += content
-
- if chunk['done'] is True:
- completion_tokens = chunk["eval_count"]
-
- yield chunk
-
- # Handling exception ensure observability without disrupting operation
- try:
- prompt_tokens = general_tokens(kwargs.get("prompt", ""))
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- # pylint: disable=line-too-long
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
+ awaited_wrapped = await wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
# Handling for non-streaming responses
else:
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
response = await wrapped(*args, **kwargs)
+ end_time = time.time()
+
+ response_dict = response_as_dict(response)
try:
- # Set base span attribues
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ input_tokens = response_dict.get('prompt_eval_count')
+ output_tokens = response_dict.get('eval_count')
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, input_tokens,
+ output_tokens)
+
+ # Set base span attribues (OTel Semconv)
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+
+ # List of attributes and their config keys
+ attributes = [
+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),
+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),
+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),
+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),
+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),
+ ]
+
+ # Safely get the options dictionary from kwargs
+ options = kwargs.get('options', {})
+
+ # Set each attribute if the corresponding value exists and is not None
+ for attribute, key in attributes:
+ # Use dictionary `get` to retrieve values from the options dictionary
+ value = options.get(key)
+ if value is not None:
+ span.set_attribute(attribute, value)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [response_dict.get('done_reason')])
+ if kwargs.get('format'):
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
+ else:
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
+
+ # Set base span attribues (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ input_tokens + output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ end_time - start_time)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['response'],
+ # pylint: disable=line-too-long
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('message').get('content')),
},
)
-
- prompt_tokens = response["prompt_eval_count"]
- completion_tokens = response["eval_count"]
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response["done_reason"]])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
+ if kwargs.get('tools'):
+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,
+ str(response_dict.get('message').get('tool_calls')))
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
+ )
+ metrics["genai_client_usage_tokens"].record(
+ input_tokens + output_tokens, attributes
+ )
+ metrics["genai_client_operation_duration"].record(
+ end_time - start_time, attributes
+ )
+ metrics["genai_server_ttft"].record(
+ end_time - start_time, attributes
+ )
metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
metrics["genai_cost"].record(cost, attributes)
# Return original response
@@ -503,13 +454,12 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,
return wrapper
-def async_embeddings(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def async_embeddings(version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for embeddings to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
application_name: Name of the application using the Ollama API.
@@ -538,71 +488,89 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,
The response from the original 'embeddings' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ server_address, server_port = set_server_address_and_port(instance, '127.0.0.1', 11434)
+ request_model = kwargs.get('model', 'all-minilm')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'
+
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
response = await wrapped(*args, **kwargs)
+ end_time = time.time()
try:
- prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ input_tokens = general_tokens(str(kwargs.get('prompt')))
+
# Calculate cost of the operation
- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
- pricing_info, prompt_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ cost = get_embed_model_cost(request_model,
+ pricing_info, input_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+
+ # Set Span attributes (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get('model', "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- prompt_tokens)
+ input_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('prompt', '')),
},
)
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get('model', "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(prompt_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=request_model,
+ )
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
index ae5707bf..05f3b74a 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
@@ -1,29 +1,33 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
"""
Module for monitoring Ollama API calls.
"""
import logging
+import time
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
from openlit.__helpers import (
+ get_chat_model_cost,
+ get_embed_model_cost,
handle_exception,
+ response_as_dict,
general_tokens,
- get_chat_model_cost,
- get_embed_model_cost
+ calculate_ttft,
+ calculate_tbt,
+ create_metrics_attributes,
+ set_server_address_and_port
)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def chat(gen_ai_endpoint, version, environment, application_name,
+def chat(version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
- Generates a telemetry wrapper for chat to collect metrics.
+ Generates a telemetry wrapper for chat completions to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
application_name: Name of the application using the Ollama API.
@@ -32,464 +36,410 @@ def chat(gen_ai_endpoint, version, environment, application_name,
trace_content: Flag indicating whether to trace the actual content.
Returns:
- A function that wraps the chat method to add telemetry.
+ A function that wraps the chat completions method to add telemetry.
"""
- def wrapper(wrapped, instance, args, kwargs):
+ class TracedSyncStream:
"""
- Wraps the 'chat' API call to add telemetry.
-
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
+ Wrapper for streaming responses to collect metrics and trace data.
+ Wraps the response to collect message IDs and aggregated response.
- Args:
- wrapped: The original 'chat' method to be wrapped.
- instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'chat' method.
- kwargs: Keyword arguments for the 'chat' method.
+ This class implements the '__aiter__' and '__anext__' methods that
+ handle asynchronous streaming responses.
- Returns:
- The response from the original 'chat' method.
+ This class also implements '__aenter__' and '__aexit__' methods that
+ handle asynchronous context management protocol.
"""
-
- # Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
-
- # pylint: disable=no-else-return
- if streaming:
- # Special handling for streaming response to accommodate the nature of data flow
- def stream_generator():
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- # Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- for chunk in wrapped(*args, **kwargs):
- # Collect aggregated response from events
- content = chunk['message']['content']
- llmresponse += content
-
- if chunk['done'] is True:
- completion_tokens = chunk["eval_count"]
-
- yield chunk
-
- # Handling exception ensure observability without disrupting operation
- try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- prompt_tokens = general_tokens(prompt)
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
-
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
-
- # Handling for non-streaming responses
- else:
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- response = wrapped(*args, **kwargs)
-
+ def __init__(
+ self,
+ wrapped,
+ span,
+ kwargs,
+ server_address,
+ server_port,
+ **args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
+ # Placeholder for aggregating streaming response
+ self._llmresponse = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._input_tokens = 0
+ self._output_tokens = 0
+
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.time()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ def __enter__(self):
+ self.__wrapped__.__enter__()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+
+ def __iter__(self):
+ return self
+
+ def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(self.__wrapped__, name)
+
+ def __next__(self):
+ try:
+ chunk = self.__wrapped__.__next__()
+ end_time = time.time()
+ # Record the timestamp for the current chunk
+ self._timestamps.append(end_time)
+
+ if len(self._timestamps) == 1:
+ # Calculate time to first chunk
+ self._ttft = calculate_ttft(self._timestamps, self._start_time)
+
+ chunked = response_as_dict(chunk)
+ self._llmresponse += chunked.get('message').get('content')
+
+ if chunked.get('eval_count'):
+ self._input_tokens = chunked.get('prompt_eval_count')
+ self._output_tokens = chunked.get('eval_count')
+ self._response_model = chunked.get('model')
+ self._finish_reason = chunked.get('done_reason')
+ return chunk
+ except StopIteration:
+ # Handling exception ensure observability without disrupting operation
try:
+ self._end_time = time.time()
+ if len(self._timestamps) > 1:
+ self._tbt = calculate_tbt(self._timestamps)
+
# Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
+ message_prompt = self._kwargs.get("messages", "")
formatted_messages = []
for message in message_prompt:
role = message["role"]
content = message["content"]
if isinstance(content, list):
- content_str = ", ".join(
- # pylint: disable=line-too-long
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
+ content_str_list = []
+ for item in content:
+ if item["type"] == "text":
+ content_str_list.append(f'text: {item["text"]}')
+ elif (item["type"] == "image_url" and
+ not item["image_url"]["url"].startswith("data:")):
+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')
+ content_str = ", ".join(content_str_list)
formatted_messages.append(f"{role}: {content_str}")
else:
formatted_messages.append(f"{role}: {content}")
prompt = "\n".join(formatted_messages)
- # Set base span attribues
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
+ request_model = self._kwargs.get("model", "gpt-4o")
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, self._input_tokens,
+ self._output_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,
+ self._server_port)
+
+ # List of attributes and their config keys
+ attributes = [
+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),
+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),
+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),
+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),
+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),
+ ]
+
+ # Safely get the options dictionary from kwargs
+ options = self._kwargs.get('options', {})
+
+ # Set each attribute if the corresponding value exists and is not None
+ for attribute, key in attributes:
+ # Use dictionary `get` to retrieve values from the options dictionary
+ value = options.get(key)
+ if value is not None:
+ self._span.set_attribute(attribute, value)
+
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [self._finish_reason])
+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ self._response_model)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ self._input_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ self._server_address)
+ if isinstance(self._llmresponse, str):
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ "text")
+ else:
+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ "json")
+
+ # Set Span attributes (Extra)
+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
- span.set_attribute(SERVICE_NAME,
+ self._span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- False)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ self._input_tokens + self._output_tokens)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
+ self._tbt)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ self._ttft)
+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
- span.add_event(
+ self._span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
- span.add_event(
+ self._span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['message']['content'],
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
},
)
-
- prompt_tokens = general_tokens(prompt)
- completion_tokens = response["eval_count"]
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response["done_reason"]])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
-
- span.set_status(Status(StatusCode.OK))
+ self._span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=self._server_address,
+ server_port=self._server_port,
+ response_model=self._response_model,
+ )
+ metrics["genai_client_usage_tokens"].record(
+ self._input_tokens + self._output_tokens, attributes
+ )
+ metrics["genai_client_operation_duration"].record(
+ self._end_time - self._start_time, attributes
+ )
+ metrics["genai_server_tbt"].record(
+ self._tbt, attributes
+ )
+ metrics["genai_server_ttft"].record(
+ self._ttft, attributes
+ )
metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)
metrics["genai_cost"].record(cost, attributes)
- # Return original response
- return response
-
except Exception as e:
- handle_exception(span, e)
+ handle_exception(self._span, e)
logger.error("Error in trace creation: %s", e)
-
- # Return original response
- return response
-
- return wrapper
-
-def generate(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
- """
- Generates a telemetry wrapper for generate to collect metrics.
-
- Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
- version: Version of the monitoring package.
- environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Ollama API.
- tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of Ollama usage.
- trace_content: Flag indicating whether to trace the actual content.
-
- Returns:
- A function that wraps the generate method to add telemetry.
- """
+ finally:
+ self._span.end()
+ raise
def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'generate' API call to add telemetry.
-
+ Wraps the 'chat.completions' API call to add telemetry.
+
This collects metrics such as execution time, cost, and token usage, and handles errors
gracefully, adding details to the trace for observability.
Args:
- wrapped: The original 'generate' method to be wrapped.
+ wrapped: The original 'chat.completions' method to be wrapped.
instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'generate' method.
- kwargs: Keyword arguments for the 'generate' method.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
Returns:
- The response from the original 'generate' method.
+ The response from the original 'chat.completions' method.
"""
# Check if streaming is enabled for the API call
streaming = kwargs.get("stream", False)
+ server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)
+ request_model = kwargs.get("model", "gpt-4o")
+
+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
# pylint: disable=no-else-return
if streaming:
# Special handling for streaming response to accommodate the nature of data flow
- def stream_generator():
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
- # Placeholder for aggregating streaming response
- llmresponse = ""
-
- # Loop through streaming events capturing relevant details
- for chunk in wrapped(*args, **kwargs):
- # Collect aggregated response from events
- content = chunk['response']
- llmresponse += content
-
- if chunk['done'] is True:
- completion_tokens = chunk["eval_count"]
-
- yield chunk
-
- # Handling exception ensure observability without disrupting operation
- try:
- prompt_tokens = general_tokens(kwargs.get("prompt", ""))
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- if trace_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- # pylint: disable=line-too-long
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,
- },
- )
+ awaited_wrapped = wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- return stream_generator()
+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
# Handling for non-streaming responses
else:
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
response = wrapped(*args, **kwargs)
+ end_time = time.time()
+
+ response_dict = response_as_dict(response)
try:
- # Set base span attribues
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ input_tokens = response_dict.get('prompt_eval_count')
+ output_tokens = response_dict.get('eval_count')
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(request_model,
+ pricing_info, input_tokens,
+ output_tokens)
+
+ # Set base span attribues (OTel Semconv)
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+
+ # List of attributes and their config keys
+ attributes = [
+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),
+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),
+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),
+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),
+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),
+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),
+ ]
+
+ # Safely get the options dictionary from kwargs
+ options = kwargs.get('options', {})
+
+ # Set each attribute if the corresponding value exists and is not None
+ for attribute, key in attributes:
+ # Use dictionary `get` to retrieve values from the options dictionary
+ value = options.get(key)
+ if value is not None:
+ span.set_attribute(attribute, value)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ response_dict.get('model'))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ [response_dict.get('done_reason')])
+ if kwargs.get('format'):
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'json')
+ else:
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text')
+
+ # Set base span attribues (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "llama3"))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ input_tokens + output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
+ end_time - start_time)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
},
)
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['response'],
+ # pylint: disable=line-too-long
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('message').get('content')),
},
)
-
- prompt_tokens = response["prompt_eval_count"]
- completion_tokens = response["eval_count"]
- total_tokens = prompt_tokens + completion_tokens
- # Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "llama3"),
- pricing_info, prompt_tokens, completion_tokens)
-
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- completion_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- total_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response["done_reason"]])
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
+ if kwargs.get('tools'):
+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,
+ str(response_dict.get('message').get('tool_calls')))
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get("model", "llama3")
- }
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=response_dict.get('model'),
+ )
+ metrics["genai_client_usage_tokens"].record(
+ input_tokens + output_tokens, attributes
+ )
+ metrics["genai_client_operation_duration"].record(
+ end_time - start_time, attributes
+ )
+ metrics["genai_server_ttft"].record(
+ end_time - start_time, attributes
+ )
metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(total_tokens, attributes)
- metrics["genai_completion_tokens"].add(completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
metrics["genai_cost"].record(cost, attributes)
# Return original response
@@ -504,13 +454,12 @@ def generate(gen_ai_endpoint, version, environment, application_name,
return wrapper
-def embeddings(gen_ai_endpoint, version, environment, application_name,
- tracer, pricing_info, trace_content, metrics, disable_metrics):
+def embeddings(version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for embeddings to collect metrics.
Args:
- gen_ai_endpoint: Endpoint identifier for logging and tracing.
version: Version of the monitoring package.
environment: Deployment environment (e.g., production, staging).
application_name: Name of the application using the Ollama API.
@@ -539,71 +488,89 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,
The response from the original 'embeddings' method.
"""
- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ server_address, server_port = set_server_address_and_port(instance, '127.0.0.1', 11434)
+ request_model = kwargs.get('model', 'all-minilm')
+
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'
+
+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
+ start_time = time.time()
response = wrapped(*args, **kwargs)
+ end_time = time.time()
try:
- prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ input_tokens = general_tokens(str(kwargs.get('prompt')))
+
# Calculate cost of the operation
- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
- pricing_info, prompt_tokens)
- # Set Span attributes
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ cost = get_embed_model_cost(request_model,
+ pricing_info, input_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)
- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
- gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
+ request_model)
+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
+ server_address)
+ span.set_attribute(SemanticConvetion.SERVER_PORT,
+ server_port)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
+ input_tokens)
+
+ # Set Span attributes (Extras)
span.set_attribute(DEPLOYMENT_ENVIRONMENT,
environment)
span.set_attribute(SERVICE_NAME,
application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get('model', "llama3"))
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- prompt_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- prompt_tokens)
+ input_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
+ version)
+
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('prompt', '')),
},
)
span.set_status(Status(StatusCode.OK))
if disable_metrics is False:
- attributes = {
- TELEMETRY_SDK_NAME:
- "openlit",
- SERVICE_NAME:
- application_name,
- SemanticConvetion.GEN_AI_SYSTEM:
- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
- DEPLOYMENT_ENVIRONMENT:
- environment,
- SemanticConvetion.GEN_AI_OPERATION:
- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
- SemanticConvetion.GEN_AI_REQUEST_MODEL:
- kwargs.get('model', "llama3")
- }
-
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(prompt_tokens, attributes)
- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+ attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,
+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ request_model=request_model,
+ server_address=server_address,
+ server_port=server_port,
+ response_model=request_model,
+ )
+ metrics['genai_client_usage_tokens'].record(
+ input_tokens, attributes
+ )
+ metrics['genai_client_operation_duration'].record(
+ end_time - start_time, attributes
+ )
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
# Return original response
return response
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
# Return original response
return response
diff --git a/sdk/python/src/openlit/instrumentation/premai/premai.py b/sdk/python/src/openlit/instrumentation/premai/premai.py
index 2af15c8d..4baea0a8 100644
--- a/sdk/python/src/openlit/instrumentation/premai/premai.py
+++ b/sdk/python/src/openlit/instrumentation/premai/premai.py
@@ -108,7 +108,6 @@ def chat(gen_ai_endpoint, version, environment, application_name,
cost = get_chat_model_cost(self._kwargs.get("model", "gpt-4o-mini"),
pricing_info, prompt_tokens,
completion_tokens)
- print(self._kwargs)
# Set Span attributes
self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,