Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Align AI21 Instrumentation to OTel Semconv (#619)
Patcherd2c2f30e77c8
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex dde0383f..9e20da8b 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.9"+version = "1.33.8"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/ai21/__init__.py b/sdk/python/src/openlit/instrumentation/ai21/__init__.pyindex 9c461e42..97099bbe 100644--- a/sdk/python/src/openlit/instrumentation/ai21/__init__.py+++ b/sdk/python/src/openlit/instrumentation/ai21/__init__.py@@ -37,13 +37,13 @@ class AI21Instrumentor(BaseInstrumentor):wrap_function_wrapper("ai21.clients.studio.resources.chat.chat_completions","ChatCompletions.create",- chat("ai21.chat.completions", version, environment, application_name,+ chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("ai21.clients.studio.resources.studio_conversational_rag","StudioConversationalRag.create",- chat_rag("ai21.conversational_rag", version, environment, application_name,+ chat_rag(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -51,13 +51,13 @@ class AI21Instrumentor(BaseInstrumentor):wrap_function_wrapper("ai21.clients.studio.resources.chat.async_chat_completions","AsyncChatCompletions.create",- async_chat("ai21.chat.completions", version, environment, application_name,+ async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)wrap_function_wrapper("ai21.clients.studio.resources.studio_conversational_rag","AsyncStudioConversationalRag.create",- async_chat_rag("ai21.conversational_rag", version, environment, application_name,+ async_chat_rag(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/ai21/ai21.py b/sdk/python/src/openlit/instrumentation/ai21/ai21.pyindex dacd4b14..8f093805 100644--- a/sdk/python/src/openlit/instrumentation/ai21/ai21.py+++ b/sdk/python/src/openlit/instrumentation/ai21/ai21.py@@ -1,15 +1,19 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes, inconsistent-return-statements"""Module for monitoring AI21 calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (get_chat_model_cost,handle_exception,response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port,general_tokens)from openlit.semcov import SemanticConvetion@@ -17,13 +21,12 @@ from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def chat(gen_ai_endpoint, version, environment, application_name,+def chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the AI21 SDK.@@ -38,6 +41,7 @@ def chat(gen_ai_endpoint, version, environment, application_name,class TracedSyncStream:"""Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'ai21.AsyncStream' response to collect message IDs and aggregated response.This class implements the '__aiter__' and '__anext__' methods thathandle asynchronous streaming responses.@@ -50,6 +54,8 @@ def chat(gen_ai_endpoint, version, environment, application_name,wrapped,span,kwargs,+ server_address,+ server_port,**args,):self.__wrapped__ = wrapped@@ -57,11 +63,19 @@ def chat(gen_ai_endpoint, version, environment, application_name,# Placeholder for aggregating streaming responseself._llmresponse = ""self._response_id = ""- self._prompt_tokens = 0- self._completion_tokens = 0+ self._finish_reason = ""+ self._input_tokens = 0+ self._output_tokens = 0self._args = argsself._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_portdef __enter__(self):self.__wrapped__.__enter__()@@ -80,8 +94,15 @@ def chat(gen_ai_endpoint, version, environment, application_name,def __next__(self):try:chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)+chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from eventsif (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and'content' in chunked.get('choices')[0].get('delta'))):@@ -90,14 +111,19 @@ def chat(gen_ai_endpoint, version, environment, application_name,self._llmresponse += contentif chunked.get('usage'):- self._prompt_tokens = chunked.get('usage').get("prompt_tokens")- self._completion_tokens = chunked.get('usage').get("completion_tokens")+ self._input_tokens = chunked.get('usage').get("prompt_tokens")+ self._output_tokens = chunked.get('usage').get("completion_tokens")self._response_id = chunked.get('id')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')return chunkexcept StopIteration:# Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single stringmessage_prompt = self._kwargs.get("messages", "")formatted_messages = []@@ -107,7 +133,6 @@ def chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -117,43 +142,74 @@ def chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)+ request_model = self._kwargs.get("model", "jamba-1.5-mini")+# Calculate cost of the operation- cost = get_chat_model_cost(self._kwargs.get("model", "jamba-1.5-mini"),- pricing_info, self._prompt_tokens,- self._completion_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)- # Set Span attributes+ # Set Span attributes (OTel Semconv)self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- self._kwargs.get("model", "jamba-1.5-mini"))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)+ self._kwargs.get("temperature", 0.4))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- self._prompt_tokens)+ self._input_tokens)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- self._completion_tokens)+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)++ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- self._prompt_tokens + self._completion_tokens)+ self._input_tokens + self._output_tokens)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -167,31 +223,35 @@ def chat(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- self._kwargs.get("model", "jamba-1.5-mini")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=request_model,+ )- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- self._prompt_tokens + self._completion_tokens, attributes+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes)- metrics["genai_completion_tokens"].add(self._completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._prompt_tokens, attributes)+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)except Exception as e:@@ -220,20 +280,25 @@ def chat(gen_ai_endpoint, version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)+ request_model = kwargs.get("model", "jamba-1.5-mini")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = wrapped(*args, **kwargs)- span = tracer.start_span(gen_ai_endpoint, kind=SpanKind.CLIENT)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedSyncStream(awaited_wrapped, span, kwargs)+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()response_dict = response_as_dict(response)@@ -247,7 +312,6 @@ def chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -257,30 +321,64 @@ def chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "jamba-1.5-mini"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get("presence_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))+ kwargs.get("temperature", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -289,93 +387,54 @@ def chat(gen_ai_endpoint, version, environment, application_name,},)- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage', {}).get('prompt_tokens', None),- response_dict.get('usage', {}).get('completion_tokens', None))-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response_dict.get('usage', {}).get('prompt_tokens', None))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response_dict.get('usage', {}).get('completion_tokens', None))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response_dict.get('usage', {}).get('total_tokens', None))+ for i in range(kwargs.get('n',1)):span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices', [])[0].get('finish_reason', None)])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices', [])[0].get("message").get("content"),- },- )-- # Set span attributes for when n > 0- else:- i = 0- while i < kwargs["n"] and trace_content is True:- attribute_name = f"gen_ai.content.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices')[i].get("message").get("content"),- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage').get('prompt_tokens'),- response_dict.get('usage').get('completion_tokens'))- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: "Function called with tools",- },- )- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response_dict.get('usage').get('prompt_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response_dict.get('usage').get('completion_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response_dict.get('usage').get('total_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "jamba-1.5-mini")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response_dict.get('usage').get('total_tokens'), attributes)- metrics["genai_completion_tokens"].add(response_dict.get('usage').get('completion_tokens'), attributes)- metrics["genai_prompt_tokens"].add(response_dict.get('usage').get('prompt_tokens'), attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original response@@ -390,13 +449,12 @@ def chat(gen_ai_endpoint, version, environment, application_name,return wrapper-def chat_rag(gen_ai_endpoint, version, environment, application_name,+def chat_rag(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the AI21 SDK.@@ -425,180 +483,173 @@ def chat_rag(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat.completions' method."""- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)-- # pylint: disable=no-else-return- if streaming:- # # Special handling for streaming response to accommodate the nature of data flow- # awaited_wrapped = wrapped(*args, **kwargs)- # span = tracer.start_span(gen_ai_endpoint, kind=SpanKind.CLIENT)-- # return TracedSyncStream(awaited_wrapped, span, kwargs)-- return+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)+ request_model = kwargs.get("model", "jamba-1.5-mini")- # Handling for non-streaming responses- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = wrapped(*args, **kwargs)-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message.role- content = message.content+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)+ end_time = time.time()- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "jamba-1.5-mini"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS,- kwargs.get("max_segments", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_STRATEGY,- kwargs.get("retrieval_strategy", "segments"))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD,- kwargs.get("retrieval_similarity_threshold", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS,- kwargs.get("max_neighbors", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_FILE_IDS,- str(kwargs.get("file_ids", "")))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH,- kwargs.get("path", ""))+ response_dict = response_as_dict(response)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },+ try:+ # Format 'messages' into a single string+ message_prompt = kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message.role+ content = message.content++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content)- prompt_tokens = general_tokens(prompt)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ input_tokens = general_tokens(prompt)++ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get("max_tokens", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get("presence_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS,+ kwargs.get("max_segments", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_STRATEGY,+ kwargs.get("retrieval_strategy", "segments"))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD,+ kwargs.get("retrieval_similarity_threshold", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS,+ kwargs.get("max_neighbors", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_FILE_IDS,+ str(kwargs.get("file_ids", "")))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH,+ kwargs.get("path", ""))+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )++ output_tokens = 0+ for i in range(kwargs.get('n',1)):+ output_tokens += general_tokens(response_dict.get('choices')[i].get('content'))-- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- prompt_tokens = general_tokens(prompt)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- completion_tokens = general_tokens(response_dict.get('choices', [])[0].get("content"))- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices', [])[0].get("content"),- },- )-- # Set span attributes for when n > 0- else:- i = 0- completion_tokens = 0- while i < kwargs["n"] and trace_content is True:- completion_tokens += general_tokens(response_dict.get('choices')[i].get("message").get("content"))- attribute_name = f"gen_ai.content.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices')[i].get("message").get("content"),- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- completion_tokens = -1- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage').get('prompt_tokens'),- response_dict.get('usage').get('completion_tokens'))+ if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: "Function called with tools",+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('content')),},)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, prompt_tokens,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "jamba-1.5-mini")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.pyindex 07675a40..9bc6c034 100644--- a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py+++ b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py@@ -1,15 +1,19 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes, inconsistent-return-statements"""Module for monitoring AI21 calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (get_chat_model_cost,handle_exception,response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port,general_tokens)from openlit.semcov import SemanticConvetion@@ -17,13 +21,12 @@ from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_chat(gen_ai_endpoint, version, environment, application_name,+def async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the AI21 SDK.@@ -38,6 +41,7 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,class TracedAsyncStream:"""Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'ai21.AsyncStream' response to collect message IDs and aggregated response.This class implements the '__aiter__' and '__anext__' methods thathandle asynchronous streaming responses.@@ -50,6 +54,8 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,wrapped,span,kwargs,+ server_address,+ server_port,**args,):self.__wrapped__ = wrapped@@ -57,11 +63,19 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,# Placeholder for aggregating streaming responseself._llmresponse = ""self._response_id = ""- self._prompt_tokens = 0- self._completion_tokens = 0+ self._finish_reason = ""+ self._input_tokens = 0+ self._output_tokens = 0self._args = argsself._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_portasync def __aenter__(self):await self.__wrapped__.__aenter__()@@ -80,8 +94,15 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)+chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from eventsif (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and'content' in chunked.get('choices')[0].get('delta'))):@@ -90,14 +111,19 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,self._llmresponse += contentif chunked.get('usage'):- self._prompt_tokens = chunked.get('usage').get("prompt_tokens")- self._completion_tokens = chunked.get('usage').get("completion_tokens")+ self._input_tokens = chunked.get('usage').get("prompt_tokens")+ self._output_tokens = chunked.get('usage').get("completion_tokens")self._response_id = chunked.get('id')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')return chunkexcept StopAsyncIteration:# Handling exception ensure observability without disrupting operationtry:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)+# Format 'messages' into a single stringmessage_prompt = self._kwargs.get("messages", "")formatted_messages = []@@ -107,7 +133,6 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -117,43 +142,74 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)+ request_model = self._kwargs.get("model", "jamba-1.5-mini")+# Calculate cost of the operation- cost = get_chat_model_cost(self._kwargs.get("model", "jamba-1.5-mini"),- pricing_info, self._prompt_tokens,- self._completion_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)- # Set Span attributes+ # Set Span attributes (OTel Semconv)self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- self._kwargs.get("model", "jamba-1.5-mini"))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)+ self._kwargs.get("temperature", 0.4))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- self._prompt_tokens)+ self._input_tokens)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- self._completion_tokens)+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)++ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- self._prompt_tokens + self._completion_tokens)+ self._input_tokens + self._output_tokens)self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -167,31 +223,35 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,},)-self._span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- self._kwargs.get("model", "jamba-1.5-mini")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=request_model,+ )- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- self._prompt_tokens + self._completion_tokens, attributes+ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes)- metrics["genai_completion_tokens"].add(self._completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._prompt_tokens, attributes)+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)except Exception as e:@@ -220,19 +280,25 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)+ request_model = kwargs.get("model", "jamba-1.5-mini")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"+# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flowawaited_wrapped = await wrapped(*args, **kwargs)- span = tracer.start_span(gen_ai_endpoint, kind=SpanKind.CLIENT)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedAsyncStream(awaited_wrapped, span, kwargs)+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()response_dict = response_as_dict(response)@@ -246,7 +312,6 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -256,30 +321,64 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "jamba-1.5-mini"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get("presence_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))+ kwargs.get("temperature", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -288,93 +387,54 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,},)- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage', {}).get('prompt_tokens', None),- response_dict.get('usage', {}).get('completion_tokens', None))-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response_dict.get('usage', {}).get('prompt_tokens', None))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response_dict.get('usage', {}).get('completion_tokens', None))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response_dict.get('usage', {}).get('total_tokens', None))+ for i in range(kwargs.get('n',1)):span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices', [])[0].get('finish_reason', None)])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices', [])[0].get("message").get("content"),- },- )-- # Set span attributes for when n > 0- else:- i = 0- while i < kwargs["n"] and trace_content is True:- attribute_name = f"gen_ai.content.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices')[i].get("message").get("content"),- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage').get('prompt_tokens'),- response_dict.get('usage').get('completion_tokens'))- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: "Function called with tools",- },- )- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response_dict.get('usage').get('prompt_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response_dict.get('usage').get('completion_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response_dict.get('usage').get('total_tokens'))- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "jamba-1.5-mini")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response_dict.get('usage').get('total_tokens'), attributes)- metrics["genai_completion_tokens"].add(response_dict.get('usage').get('completion_tokens'), attributes)- metrics["genai_prompt_tokens"].add(response_dict.get('usage').get('prompt_tokens'), attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original response@@ -389,13 +449,12 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,return wrapper-def async_chat_rag(gen_ai_endpoint, version, environment, application_name,+def async_chat_rag(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the AI21 SDK.@@ -424,180 +483,173 @@ def async_chat_rag(gen_ai_endpoint, version, environment, application_name,The response from the original 'chat.completions' method."""- # Check if streaming is enabled for the API call- streaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)+ request_model = kwargs.get("model", "jamba-1.5-mini")- # pylint: disable=no-else-return- if streaming:- # # Special handling for streaming response to accommodate the nature of data flow- # awaited_wrapped = wrapped(*args, **kwargs)- # span = tracer.start_span(gen_ai_endpoint, kind=SpanKind.CLIENT)+ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # return TracedSyncStream(awaited_wrapped, span, kwargs)+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()+ response = await wrapped(*args, **kwargs)+ end_time = time.time()- return+ response_dict = response_as_dict(response)- # Handling for non-streaming responses- else:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- response = await wrapped(*args, **kwargs)-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message.role- content = message.content-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_AI21)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "jamba-1.5-mini"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS,- kwargs.get("max_segments", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_STRATEGY,- kwargs.get("retrieval_strategy", "segments"))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD,- kwargs.get("retrieval_similarity_threshold", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS,- kwargs.get("max_neighbors", -1))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_FILE_IDS,- str(kwargs.get("file_ids", "")))- span.set_attribute(SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH,- kwargs.get("path", ""))-- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },+ try:+ # Format 'messages' into a single string+ message_prompt = kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message.role+ content = message.content++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content)- prompt_tokens = general_tokens(prompt)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ input_tokens = general_tokens(prompt)++ # Set base span attribues (OTel Semconv)+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_AI21)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get("max_tokens", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get("presence_penalty", 0.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS,+ kwargs.get("max_segments", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_STRATEGY,+ kwargs.get("retrieval_strategy", "segments"))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD,+ kwargs.get("retrieval_similarity_threshold", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS,+ kwargs.get("max_neighbors", -1))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_FILE_IDS,+ str(kwargs.get("file_ids", "")))+ span.set_attribute(SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH,+ kwargs.get("path", ""))+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )++ output_tokens = 0+ for i in range(kwargs.get('n',1)):+ output_tokens += general_tokens(response_dict.get('choices')[i].get('content'))-- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- prompt_tokens = general_tokens(prompt)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- completion_tokens = general_tokens(response_dict.get('choices', [])[0].get("content"))- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices', [])[0].get("content"),- },- )-- # Set span attributes for when n > 0- else:- i = 0- completion_tokens = 0- while i < kwargs["n"] and trace_content is True:- completion_tokens += general_tokens(response_dict.get('choices')[i].get("message").get("content"))- attribute_name = f"gen_ai.content.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response_dict.get('choices')[i].get("message").get("content"),- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- completion_tokens = -1- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, response_dict.get('usage').get('prompt_tokens'),- response_dict.get('usage').get('completion_tokens'))+ if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: "Function called with tools",+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('content')),},)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "jamba-1.5-mini"),- pricing_info, prompt_tokens,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_AI21,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "jamba-1.5-mini")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- # Return original response- return response+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_AI21,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.pyindex def04ed2..ac9b78d7 100644--- a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py@@ -341,7 +341,6 @@ def async_chat_stream(version, environment, application_name,Generates a telemetry wrapper for chat_stream to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Cohere API.diff --git a/sdk/python/src/openlit/instrumentation/cohere/cohere.py b/sdk/python/src/openlit/instrumentation/cohere/cohere.pyindex 9b772f61..7158c604 100644--- a/sdk/python/src/openlit/instrumentation/cohere/cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/cohere.py@@ -341,7 +341,6 @@ def chat_stream(version, environment, application_name,Generates a telemetry wrapper for chat_stream to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Cohere API.diff --git a/sdk/python/tests/test_ai21.py.hold b/sdk/python/tests/test_ai21.pysimilarity index 98%rename from sdk/python/tests/test_ai21.py.holdrename to sdk/python/tests/test_ai21.pyindex bd1ce73f..32f76071 100644--- a/sdk/python/tests/test_ai21.py.hold+++ b/sdk/python/tests/test_ai21.py@@ -27,7 +27,7 @@ async_client = AsyncAI21Client()openlit.init(environment="openlit-python-testing", application_name="openlit-python-ai21-test")MESSAGES = [- ChatMessage(content="say hi", role="user"),+ ChatMessage(content="hi", role="user"),]def test_sync_ai21_chat():