Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update Mistral and Ollama OpenTelemetry Instrumentation and align with Semantic Conventions (#628)
Patcherdb8d31da9ffb
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex dde0383f..eea7ee41 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.9"+version = "1.33.10"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 01a12c6c..5a7de73a 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -266,6 +266,11 @@ def set_server_address_and_port(client_instance: Any,config = getattr(client_instance, "_config", None)base_url = getattr(config, "endpoint", None)+ if not base_url:+ # Attempt to get server_url from instance.sdk_configuration.server_url+ config = getattr(client_instance, "sdk_configuration", None)+ base_url = getattr(config, "server_url", None)+if base_url:if isinstance(base_url, str):url = urlparse(base_url)diff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.pyindex 54440aec..2a375d81 100644--- a/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/async_elevenlabs.py@@ -60,8 +60,7 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,start_time = time.time()response = await wrapped(*args, **kwargs)end_time = time.time()- for key, value in instance.__dict__.items():- print(f'{key}: {value}')+try:# Calculate cost of the operationcost = get_audio_model_cost(request_model,diff --git a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.pyindex 7b3d9cec..4dc6821a 100644--- a/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py+++ b/sdk/python/src/openlit/instrumentation/elevenlabs/elevenlabs.py@@ -61,8 +61,7 @@ def generate(gen_ai_endpoint, version, environment, application_name,start_time = time.time()response = wrapped(*args, **kwargs)end_time = time.time()- for key, value in instance.__dict__.items():- print(f'{key}: {value}')+try:# Calculate cost of the operationcost = get_audio_model_cost(request_model,diff --git a/sdk/python/src/openlit/instrumentation/mistral/__init__.py b/sdk/python/src/openlit/instrumentation/mistral/__init__.pyindex 123f6e4b..7bc0074d 100644--- a/sdk/python/src/openlit/instrumentation/mistral/__init__.py+++ b/sdk/python/src/openlit/instrumentation/mistral/__init__.py@@ -31,7 +31,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.chat","Chat.complete",- chat("mistral.chat", version, environment, application_name,+ chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -39,7 +39,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.chat","Chat.stream",- chat_stream("mistral.chat", version, environment, application_name,+ chat_stream(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -47,7 +47,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.embeddings","Embeddings.create",- embeddings("mistral.embeddings", version, environment, application_name,+ embeddings(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -55,7 +55,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.chat","Chat.complete_async",- async_chat("mistral.chat", version, environment, application_name,+ async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -63,7 +63,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.chat","Chat.stream_async",- async_chat_stream("mistral.chat", version, environment, application_name,+ async_chat_stream(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -71,7 +71,7 @@ class MistralInstrumentor(BaseInstrumentor):wrap_function_wrapper("mistralai.embeddings","Embeddings.create_async",- async_embeddings("mistral.embeddings", version, environment, application_name,+ async_embeddings(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.pyindex 8eb8f635..e92bc284 100644--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py@@ -1,29 +1,37 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Mistral API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception+from openlit.__helpers import (+ get_chat_model_cost,+ get_embed_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_chat(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def async_chat(version, environment, application_name, tracer,+ pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -47,71 +55,95 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Handling exception ensure observability without disrupting operation+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-small-latest')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string- message_prompt = kwargs.get('messages', "")+ message_prompt = kwargs.get('messages', '')formatted_messages = []for message in message_prompt:- role = message["role"]- content = message["content"]+ role = message['role']+ content = message['content']if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-long- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"- if 'type' in item else f"text: {item['text']}"+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'for item in content)- formatted_messages.append(f"{role}: {content_str}")+ formatted_messages.append(f'{role}: {content_str}')else:- formatted_messages.append(f"{role}: {content}")- prompt = " ".join(formatted_messages)+ formatted_messages.append(f'{role}: {content}')+ prompt = '\n'.join(formatted_messages)++ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get('seed', ''))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get('frequency_penalty', 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get('max_tokens', -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get('presence_penalty', 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get('stop', []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get('temperature', 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get('top_p', 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)+ response_dict.get('id'))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "mistral-small-latest"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("random_seed", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0].finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -119,64 +151,80 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content if response.choices[0].message.content else "",- },- )++ for i in range(kwargs.get('n',1)):+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "mistral-small-latest")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_completion_tokens"].add(- response.usage.completion_tokens, attributes+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )++ metrics['genai_client_usage_tokens'].record(+ input_tokens + output_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_server_ttft'].record(+ end_time - start_time, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)- metrics["genai_cost"].record(cost)+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_completion_tokens'].add(output_tokens, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsereturn wrapper-def async_chat_stream(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def async_chat_stream(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat_stream to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -200,149 +248,260 @@ def async_chat_stream(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat_stream' method."""- async def stream_generator():- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ class TracedAsyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'mistral.syncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span# Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in await wrapped(*args, **kwargs):- response_id = event.data.id- llmresponse += event.data.choices[0].delta.content- if event.data.usage is not None:- prompt_tokens = event.data.usage.prompt_tokens- completion_tokens = event.data.usage.completion_tokens- total_tokens = event.data.usage.total_tokens- finish_reason = event.data.choices[0].finish_reason- yield event-- # Handling exception ensure observability without disrupting operation+ self._llmresponse = ''+ self._response_id = ''+ self._response_model = ''+ self._finish_reason = ''+ self._input_tokens = ''+ self._output_tokens = ''++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):try:- # Format 'messages' into a single string- message_prompt = kwargs.get('messages', "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"- if 'type' in item else f"text: {item['text']}"- for item in content- )- formatted_messages.append(f"{role}: {content_str}")+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)++ self._llmresponse += chunked.get('data').get('choices')[0].get('delta').get('content')+ if chunked.get('data').get('usage') is not None:+ self._response_id = chunked.get('data').get('id')+ self._response_model = chunked.get('data').get('model')+ self._input_tokens = chunked.get('data').get('usage').get('prompt_tokens')+ self._output_tokens = chunked.get('data').get('usage').get('completion_tokens')+ self._finish_reason = chunked.get('data').get('choices')[0].get('finish_reason')++ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get('messages', '')+ formatted_messages = []+ for message in message_prompt:+ role = message['role']+ content = message['content']++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item['type'] == 'text':+ content_str_list.append(f'text: {item["text"]}')+ elif (item['type'] == 'image_url' and+ not item['image_url']['url'].startswith('data:')):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f'{role}: {content_str}')+ else:+ formatted_messages.append(f'{role}: {content}')+ prompt = '\n'.join(formatted_messages)++ request_model = self._kwargs.get('model', 'mistral-small-latest')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get('seed', ''))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get('frequency_penalty', 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get('max_tokens', -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get('presence_penalty', 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get('stop_sequences', []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get('temperature', 0.3))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ self._kwargs.get('k', 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get('p', 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)++ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')else:- formatted_messages.append(f"{role}: {content}")- prompt = " ".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "mistral-small-latest"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("random_seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics['genai_client_usage_tokens'].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ self._end_time - self._start_time, attributes+ )+ metrics['genai_server_tbt'].record(+ self._tbt, attributes+ )+ metrics['genai_server_ttft'].record(+ self._ttft, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_completion_tokens'].add(self._output_tokens, attributes)+ metrics['genai_prompt_tokens'].add(self._input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error('Error in trace creation: %s', e)+ finally:+ self._span.end()+ raise++ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-small-latest')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "mistral-small-latest")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)return wrapper-def async_embeddings(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def async_embeddings(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:A function that wraps the embeddings method to add telemetry."""- async def wrapper(wrapped, instance, args, kwargs):+ def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'embeddings' API call to add telemetry.@@ -359,78 +518,92 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-embed')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:- # Get prompt from kwargs and store as a single string- prompt = ', '.join(kwargs.get('inputs', []))+ input_tokens = response_dict.get('usage').get('prompt_tokens')# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),- pricing_info, response.usage.prompt_tokens)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ # Set Span attributes (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,+ [kwargs.get('encoding_format', 'float')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)++ # Set Span attributes (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get('model', "mistral-embed"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get("encoding_format", "float"))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('inputs', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get('model', "mistral-embed")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics['genai_client_usage_tokens'].record(+ input_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/mistral/mistral.py b/sdk/python/src/openlit/instrumentation/mistral/mistral.pyindex 5b2b6583..3dd363f6 100644--- a/sdk/python/src/openlit/instrumentation/mistral/mistral.py+++ b/sdk/python/src/openlit/instrumentation/mistral/mistral.py@@ -1,29 +1,37 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Mistral API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT-from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception+from openlit.__helpers import (+ get_chat_model_cost,+ get_embed_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def chat(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def chat(version, environment, application_name, tracer,+ pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -47,70 +55,95 @@ def chat(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-small-latest')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string- message_prompt = kwargs.get('messages', "")+ message_prompt = kwargs.get('messages', '')formatted_messages = []for message in message_prompt:- role = message["role"]- content = message["content"]+ role = message['role']+ content = message['content']if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-long- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"- if 'type' in item else f"text: {item['text']}"+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'for item in content)- formatted_messages.append(f"{role}: {content_str}")+ formatted_messages.append(f'{role}: {content_str}')else:- formatted_messages.append(f"{role}: {content}")- prompt = " ".join(formatted_messages)+ formatted_messages.append(f'{role}: {content}')+ prompt = '\n'.join(formatted_messages)++ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get('seed', ''))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get('frequency_penalty', 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get('max_tokens', -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get('presence_penalty', 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get('stop', []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get('temperature', 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get('top_p', 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)+ response_dict.get('id'))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "mistral-small-latest"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("random_seed", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0].finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -118,64 +151,80 @@ def chat(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- # pylint: disable=line-too-long- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content if response.choices[0].message.content else "",- },- )++ for i in range(kwargs.get('n',1)):+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "mistral-small-latest")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_completion_tokens"].add(- response.usage.completion_tokens, attributes+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)++ metrics['genai_client_usage_tokens'].record(+ input_tokens + output_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_server_ttft'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_completion_tokens'].add(output_tokens, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsereturn wrapper-def chat_stream(gen_ai_endpoint, version, environment, application_name,+def chat_stream(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat_stream to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -199,142 +248,253 @@ def chat_stream(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat_stream' method."""- def stream_generator():- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'mistral.syncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span# Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):- response_id = event.data.id- llmresponse += event.data.choices[0].delta.content- if event.data.usage is not None:- prompt_tokens = event.data.usage.prompt_tokens- completion_tokens = event.data.usage.completion_tokens- total_tokens = event.data.usage.total_tokens- finish_reason = event.data.choices[0].finish_reason- yield event-- # Handling exception ensure observability without disrupting operation+ self._llmresponse = ''+ self._response_id = ''+ self._response_model = ''+ self._finish_reason = ''+ self._input_tokens = ''+ self._output_tokens = ''++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):try:- # Format 'messages' into a single string- message_prompt = kwargs.get('messages', "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"- if 'type' in item else f"text: {item['text']}"- for item in content- )- formatted_messages.append(f"{role}: {content_str}")+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)++ self._llmresponse += chunked.get('data').get('choices')[0].get('delta').get('content')+ if chunked.get('data').get('usage') is not None:+ self._response_id = chunked.get('data').get('id')+ self._response_model = chunked.get('data').get('model')+ self._input_tokens = chunked.get('data').get('usage').get('prompt_tokens')+ self._output_tokens = chunked.get('data').get('usage').get('completion_tokens')+ self._finish_reason = chunked.get('data').get('choices')[0].get('finish_reason')++ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get('messages', '')+ formatted_messages = []+ for message in message_prompt:+ role = message['role']+ content = message['content']++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item['type'] == 'text':+ content_str_list.append(f'text: {item["text"]}')+ elif (item['type'] == 'image_url' and+ not item['image_url']['url'].startswith('data:')):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f'{role}: {content_str}')+ else:+ formatted_messages.append(f'{role}: {content}')+ prompt = '\n'.join(formatted_messages)++ request_model = self._kwargs.get('model', 'mistral-small-latest')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get('seed', ''))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get('frequency_penalty', 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get('max_tokens', -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get('presence_penalty', 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get('stop_sequences', []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get('temperature', 0.3))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ self._kwargs.get('k', 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get('p', 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)++ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')else:- formatted_messages.append(f"{role}: {content}")- prompt = " ".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "mistral-small-latest"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("random_seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics['genai_client_usage_tokens'].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ self._end_time - self._start_time, attributes+ )+ metrics['genai_server_tbt'].record(+ self._tbt, attributes+ )+ metrics['genai_server_ttft'].record(+ self._ttft, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_completion_tokens'].add(self._output_tokens, attributes)+ metrics['genai_prompt_tokens'].add(self._input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error('Error in trace creation: %s', e)+ finally:+ self._span.end()+ raise- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "mistral-small-latest")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-small-latest')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'++ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)return wrapper-def embeddings(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def embeddings(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Mistral API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Mistral usage.trace_content: Flag indicating whether to trace the actual content.Returns:@@ -358,78 +518,92 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, 'api.mistral.ai', 443)+ request_model = kwargs.get('model', 'mistral-embed')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()+ response_dict = response_as_dict(response)try:- # Get prompt from kwargs and store as a single string- prompt = ', '.join(kwargs.get('inputs', []))+ input_tokens = response_dict.get('usage').get('prompt_tokens')# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),- pricing_info, response.usage.prompt_tokens)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ # Set Span attributes (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,+ [kwargs.get('encoding_format', 'float')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)++ # Set Span attributes (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get('model', "mistral-embed"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get("encoding_format", "float"))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('inputs', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get('model', "mistral-embed")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics['genai_client_usage_tokens'].record(+ input_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/ollama/__init__.py b/sdk/python/src/openlit/instrumentation/ollama/__init__.pyindex b32bd094..d0f68237 100644--- a/sdk/python/src/openlit/instrumentation/ollama/__init__.py+++ b/sdk/python/src/openlit/instrumentation/ollama/__init__.py@@ -7,10 +7,10 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapperfrom openlit.instrumentation.ollama.ollama import (- chat, embeddings, generate+ chat, embeddings)from openlit.instrumentation.ollama.async_ollama import (- async_chat, async_embeddings, async_generate+ async_chat, async_embeddings)_instruments = ("ollama >= 0.2.0",)@@ -37,13 +37,13 @@ class OllamaInstrumentor(BaseInstrumentor):wrap_function_wrapper("ollama","chat",- chat("ollama.chat", version, environment, application_name,+ chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("ollama","Client.chat",- chat("ollama.chat", version, environment, application_name,+ chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -51,27 +51,13 @@ class OllamaInstrumentor(BaseInstrumentor):wrap_function_wrapper("ollama","embeddings",- embeddings("ollama.embeddings", version, environment, application_name,+ embeddings(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("ollama","Client.embeddings",- embeddings("ollama.embeddings", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- # sync generate- wrap_function_wrapper(- "ollama",- "generate",- generate("ollama.generate", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )- wrap_function_wrapper(- "ollama",- "Client.generate",- generate("ollama.generate", version, environment, application_name,+ embeddings(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -79,7 +65,7 @@ class OllamaInstrumentor(BaseInstrumentor):wrap_function_wrapper("ollama","AsyncClient.chat",- async_chat("ollama.chat", version, environment, application_name,+ async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -87,15 +73,7 @@ class OllamaInstrumentor(BaseInstrumentor):wrap_function_wrapper("ollama","AsyncClient.embeddings",- async_embeddings("ollama.embeddings", version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics),- )-- # aync generate- wrap_function_wrapper(- "ollama",- "AsyncClient.generate",- async_generate("ollama.generate", version, environment, application_name,+ async_embeddings(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.pyindex 493284f7..18c9c1d1 100644--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py@@ -1,28 +1,33 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Ollama API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (+ get_chat_model_cost,+ get_embed_model_cost,handle_exception,+ response_as_dict,general_tokens,- get_chat_model_cost,- get_embed_model_cost)+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_chat(gen_ai_endpoint, version, environment, application_name,+def async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat to collect metrics.+ Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Ollama API.@@ -31,464 +36,410 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,trace_content: Flag indicating whether to trace the actual content.Returns:- A function that wraps the chat method to add telemetry.+ A function that wraps the chat completions method to add telemetry."""- async def wrapper(wrapped, instance, args, kwargs):+ class TracedAsyncStream:"""- Wraps the 'chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the response to collect message IDs and aggregated response.- Args:- wrapped: The original 'chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.- Returns:- The response from the original 'chat' method.+ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol."""-- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)-- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- # Collect aggregated response from events- content = chunk['message']['content']- llmresponse += content-- if chunk['done'] is True:- completion_tokens = chunk["eval_count"]-- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- prompt_tokens = general_tokens(prompt)- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()-- # Handling for non-streaming responses- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_model = ""+ self._finish_reason = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ self._llmresponse += chunked.get('message').get('content')++ if chunked.get('eval_count'):+ self._input_tokens = chunked.get('prompt_eval_count')+ self._output_tokens = chunked.get('eval_count')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('done_reason')+ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")+ message_prompt = self._kwargs.get("messages", "")formatted_messages = []for message in message_prompt:role = message["role"]content = message["content"]if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)formatted_messages.append(f"{role}: {content_str}")else:formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Safely get the options dictionary from kwargs+ options = self._kwargs.get('options', {})++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use dictionary `get` to retrieve values from the options dictionary+ value = options.get(key)+ if value is not None:+ self._span.set_attribute(attribute, value)++ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)- span.set_attribute(SERVICE_NAME,+ self._span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['message']['content'],+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-- prompt_tokens = general_tokens(prompt)- completion_tokens = response["eval_count"]- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response["done_reason"]])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))+ self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)- # Return original response- return response-except Exception as e:- handle_exception(span, e)+ handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def async_generate(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for generate to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Ollama API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Ollama usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the generate method to add telemetry.- """+ finally:+ self._span.end()+ raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'generate' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'generate' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate' method.- kwargs: Keyword arguments for the 'generate' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'generate' method.+ The response from the original 'chat.completions' method."""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)+ request_model = kwargs.get("model", "gpt-4o")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- # Collect aggregated response from events- content = chunk['response']- llmresponse += content-- if chunk['done'] is True:- completion_tokens = chunk["eval_count"]-- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- prompt_tokens = general_tokens(kwargs.get("prompt", ""))- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:- # Set base span attribues+ # Format 'messages' into a single string+ message_prompt = kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ input_tokens = response_dict.get('prompt_eval_count')+ output_tokens = response_dict.get('eval_count')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Safely get the options dictionary from kwargs+ options = kwargs.get('options', {})++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use dictionary `get` to retrieve values from the options dictionary+ value = options.get(key)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('done_reason')])+ if kwargs.get('format'):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')+ else:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')++ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['response'],+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('message').get('content')),},)-- prompt_tokens = response["prompt_eval_count"]- completion_tokens = response["eval_count"]- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response["done_reason"]])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('message').get('tool_calls')))span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original response@@ -503,13 +454,12 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,return wrapper-def async_embeddings(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def async_embeddings(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Ollama API.@@ -538,71 +488,89 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, '127.0.0.1', 11434)+ request_model = kwargs.get('model', 'all-minilm')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()try:- prompt_tokens = general_tokens(kwargs.get('prompt', ""))+ input_tokens = general_tokens(str(kwargs.get('prompt')))+# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),- pricing_info, prompt_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)++ # Set Span attributes (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)++ # Set Span attributes (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get('model', "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('prompt', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get('model', "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics['genai_client_usage_tokens'].record(+ input_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.pyindex ae5707bf..05f3b74a 100644--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py@@ -1,29 +1,33 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Ollama API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (+ get_chat_model_cost,+ get_embed_model_cost,handle_exception,+ response_as_dict,general_tokens,- get_chat_model_cost,- get_embed_model_cost+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def chat(gen_ai_endpoint, version, environment, application_name,+def chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat to collect metrics.+ Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Ollama API.@@ -32,464 +36,410 @@ def chat(gen_ai_endpoint, version, environment, application_name,trace_content: Flag indicating whether to trace the actual content.Returns:- A function that wraps the chat method to add telemetry.+ A function that wraps the chat completions method to add telemetry."""- def wrapper(wrapped, instance, args, kwargs):+ class TracedSyncStream:"""- Wraps the 'chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the response to collect message IDs and aggregated response.- Args:- wrapped: The original 'chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.+ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.- Returns:- The response from the original 'chat' method.+ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol."""-- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)-- # pylint: disable=no-else-return- if streaming:- # Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- # Collect aggregated response from events- content = chunk['message']['content']- llmresponse += content-- if chunk['done'] is True:- completion_tokens = chunk["eval_count"]-- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- prompt_tokens = general_tokens(prompt)- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()-- # Handling for non-streaming responses- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_model = ""+ self._finish_reason = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ self._llmresponse += chunked.get('message').get('content')++ if chunked.get('eval_count'):+ self._input_tokens = chunked.get('prompt_eval_count')+ self._output_tokens = chunked.get('eval_count')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('done_reason')+ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")+ message_prompt = self._kwargs.get("messages", "")formatted_messages = []for message in message_prompt:role = message["role"]content = message["content"]if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)formatted_messages.append(f"{role}: {content_str}")else:formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Safely get the options dictionary from kwargs+ options = self._kwargs.get('options', {})++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use dictionary `get` to retrieve values from the options dictionary+ value = options.get(key)+ if value is not None:+ self._span.set_attribute(attribute, value)++ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)- span.set_attribute(SERVICE_NAME,+ self._span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)- span.add_event(+ self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['message']['content'],+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-- prompt_tokens = general_tokens(prompt)- completion_tokens = response["eval_count"]- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response["done_reason"]])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))+ self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)- # Return original response- return response-except Exception as e:- handle_exception(span, e)+ handle_exception(self._span, e)logger.error("Error in trace creation: %s", e)-- # Return original response- return response-- return wrapper--def generate(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for generate to collect metrics.-- Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Ollama API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Ollama usage.- trace_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the generate method to add telemetry.- """+ finally:+ self._span.end()+ raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'generate' API call to add telemetry.-+ Wraps the 'chat.completions' API call to add telemetry.+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'generate' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate' method.- kwargs: Keyword arguments for the 'generate' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'generate' method.+ The response from the original 'chat.completions' method."""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)+ request_model = kwargs.get("model", "gpt-4o")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- # Collect aggregated response from events- content = chunk['response']- llmresponse += content-- if chunk['done'] is True:- completion_tokens = chunk["eval_count"]-- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- prompt_tokens = general_tokens(kwargs.get("prompt", ""))- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:- # Set base span attribues+ # Format 'messages' into a single string+ message_prompt = kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ input_tokens = response_dict.get('prompt_eval_count')+ output_tokens = response_dict.get('eval_count')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)++ # List of attributes and their config keys+ attributes = [+ (SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, 'repeat_penalty'),+ (SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, 'max_tokens'),+ (SemanticConvetion.GEN_AI_REQUEST_SEED, 'seed'),+ (SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, 'stop'),+ (SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, 'temperature'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_P, 'top_p'),+ (SemanticConvetion.GEN_AI_REQUEST_TOP_K, 'top_k'),+ ]++ # Safely get the options dictionary from kwargs+ options = kwargs.get('options', {})++ # Set each attribute if the corresponding value exists and is not None+ for attribute, key in attributes:+ # Use dictionary `get` to retrieve values from the options dictionary+ value = options.get(key)+ if value is not None:+ span.set_attribute(attribute, value)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [response_dict.get('done_reason')])+ if kwargs.get('format'):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'json')+ else:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ 'text')++ # Set base span attribues (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3"))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,},)span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response['response'],+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('message').get('content')),},)-- prompt_tokens = response["prompt_eval_count"]- completion_tokens = response["eval_count"]- total_tokens = prompt_tokens + completion_tokens- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3"),- pricing_info, prompt_tokens, completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response["done_reason"]])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('message').get('tool_calls')))span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "llama3")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original response@@ -504,13 +454,12 @@ def generate(gen_ai_endpoint, version, environment, application_name,return wrapper-def embeddings(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def embeddings(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Ollama API.@@ -539,71 +488,89 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, '127.0.0.1', 11434)+ request_model = kwargs.get('model', 'all-minilm')++ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}'++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()try:- prompt_tokens = general_tokens(kwargs.get('prompt', ""))+ input_tokens = general_tokens(str(kwargs.get('prompt')))+# Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),- pricing_info, prompt_tokens)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ cost = get_embed_model_cost(request_model,+ pricing_info, input_tokens)++ # Set Span attributes (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)++ # Set Span attributes (Extras)span.set_attribute(DEPLOYMENT_ENVIRONMENT,environment)span.set_attribute(SERVICE_NAME,application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get('model', "llama3"))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: kwargs.get("prompt", ""),+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get('prompt', '')),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SERVICE_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,- DEPLOYMENT_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get('model', "llama3")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics['genai_client_usage_tokens'].record(+ input_tokens, attributes+ )+ metrics['genai_client_operation_duration'].record(+ end_time - start_time, attributes+ )+ metrics['genai_requests'].add(1, attributes)+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)+ metrics['genai_cost'].record(cost, attributes)# Return original responsereturn responseexcept Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ logger.error('Error in trace creation: %s', e)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/premai/premai.py b/sdk/python/src/openlit/instrumentation/premai/premai.pyindex 2af15c8d..4baea0a8 100644--- a/sdk/python/src/openlit/instrumentation/premai/premai.py+++ b/sdk/python/src/openlit/instrumentation/premai/premai.py@@ -108,7 +108,6 @@ def chat(gen_ai_endpoint, version, environment, application_name,cost = get_chat_model_cost(self._kwargs.get("model", "gpt-4o-mini"),pricing_info, prompt_tokens,completion_tokens)- print(self._kwargs)# Set Span attributesself._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,