Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update Anthropic Instrumentation with to align with OTel Semconv (#617)
Patcher15a2325f0bda
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex dde0383f..9e20da8b 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.9"+version = "1.33.8"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 626b56aa..7ae739b5 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -6,7 +6,7 @@ import osimport jsonimport loggingfrom urllib.parse import urlparse-from typing import Any, Dict, List+from typing import Any, Dict, List, Tupleimport requestsimport tiktokenfrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT@@ -249,3 +249,28 @@ def create_metrics_attributes(SemanticConvetion.SERVER_PORT: server_port,SemanticConvetion.GEN_AI_RESPONSE_MODEL: response_model}++def set_server_address_and_port(client_instance: Any,+ default_server_address: str, default_server_port: int) -> Tuple[str, int]:+ """+ Determines and returns the server address and port based on the provided client's `base_url`,+ using defaults if none found or values are None.+ """++ base_client = getattr(client_instance, "_client", None)+ base_url = getattr(base_client, "base_url", None)++ if base_url:+ if isinstance(base_url, str):+ url = urlparse(base_url)+ server_address = url.hostname or default_server_address+ server_port = url.port if url.port is not None else default_server_port+ else: # base_url might not be a str; handle as an object.+ server_address = getattr(base_url, "host", None) or default_server_address+ port_attr = getattr(base_url, "port", None)+ server_port = port_attr if port_attr is not None else default_server_port+ else: # no base_url provided; use defaults.+ server_address = default_server_address+ server_port = default_server_port++ return server_address, server_portdiff --git a/sdk/python/src/openlit/instrumentation/anthropic/__init__.py b/sdk/python/src/openlit/instrumentation/anthropic/__init__.pyindex 7a833165..4079cba6 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/__init__.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/__init__.py@@ -20,8 +20,8 @@ class AnthropicInstrumentor(BaseInstrumentor):return _instrumentsdef _instrument(self, **kwargs):- application_name = kwargs.get("application_name", "default_application")- environment = kwargs.get("environment", "default_environment")+ application_name = kwargs.get("application_name", "default")+ environment = kwargs.get("environment", "default")tracer = kwargs.get("tracer")metrics = kwargs.get("metrics_dict")pricing_info = kwargs.get("pricing_info", {})@@ -33,7 +33,7 @@ class AnthropicInstrumentor(BaseInstrumentor):wrap_function_wrapper("anthropic.resources.messages","Messages.create",- messages("anthropic.messages", version, environment, application_name,+ messages(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -41,7 +41,7 @@ class AnthropicInstrumentor(BaseInstrumentor):wrap_function_wrapper("anthropic.resources.messages","AsyncMessages.create",- async_messages("anthropic.messages", version, environment, application_name,+ async_messages(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.pyindex 037e77b1..7698fb06 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py@@ -1,35 +1,267 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Anthropic API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_chat_model_cost, handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def messages(gen_ai_endpoint, version, environment, application_name, tracer,+def messages(version, environment, application_name, tracer,pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for messages to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Anthropic API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Anthropic usage.trace_content: Flag indicating whether to trace the actual content.Returns:A function that wraps the chat method to add telemetry."""+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'anthropic.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._input_tokens = ""+ self._output_tokens = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)++ # Collect message IDs and input token from events+ if chunked.get('type') == "message_start":+ self._response_id = chunked.get('message').get('id')+ self._input_tokens = chunked.get('message').get('usage').get('input_tokens')+ self._response_model = chunked.get('message').get('model')+ # Collect message IDs and aggregated response from events+ if chunked.get('type') == "content_block_delta":+ content = chunked.get('delta').get('text')+ if content:+ self._llmresponse += content+ # Collect output tokens and stop reason from events+ if chunked.get('type') == "message_delta":+ self._output_tokens = chunked.get('usage').get('output_tokens')+ self._finish_reason = chunked.get('delta').get('stop_reason')++ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop_sequences", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ self._kwargs.get("top_k", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'messages' API call to add telemetry.@@ -49,147 +281,27 @@ def messages(gen_ai_endpoint, version, environment, application_name, tracer,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.anthropic.com", 443)+ request_model = kwargs.get("model", "claude-3-5-sonnet-latest")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for event in wrapped(*args, **kwargs):-- # Collect message IDs and input token from events- if event.type == "message_start":- response_id = event.message.id- prompt_tokens = event.message.usage.input_tokens-- # Aggregate response content- if event.type == "content_block_delta":- llmresponse += event.delta.text-- # Collect output tokens and stop reason from events- if event.type == "message_delta":- completion_tokens = event.usage.output_tokens- finish_reason = event.delta.stop_reason- yield event-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(- kwargs.get("model", "claude-3-sonnet-20240229"),- pricing_info, prompt_tokens, completion_tokens- )+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "claude-3-sonnet-20240229"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", ""))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "claude-3-sonnet-20240229")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- prompt_tokens + completion_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()+ response_dict = response_as_dict(response)try:# Format 'messages' into a single stringmessage_prompt = kwargs.get("messages", "")@@ -210,48 +322,70 @@ def messages(gen_ai_endpoint, version, environment, application_name, tracer,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)+ input_tokens = response_dict.get('usage').get('input_tokens')+ output_tokens = response_dict.get('usage').get('output_tokens')+# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "claude-3-sonnet-20240229"),- pricing_info, response.usage.input_tokens,- response.usage.output_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ llm_response = ""+ for i in range(len(response_dict.get('content'))):+ if response_dict.get('content')[i].get('type') == 'text':+ llm_response = response_dict.get('content')[i].get('text')- # Set Span attribues+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "claude-3-sonnet-20240229"))+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop_sequences", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", ""))+ kwargs.get("top_k", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.stop_reason])+ [response_dict.get('stop_reason')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get('id'))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.input_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.output_tokens)+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ response_dict.get('content')[0].get('type'))++ # Set Span attributes (Extra)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.input_tokens +- response.usage.output_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -264,36 +398,36 @@ def messages(gen_ai_endpoint, version, environment, application_name, tracer,name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={# pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.content[0].text if response.content else "",+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "claude-3-sonnet-20240229")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage.input_tokens +- response.usage.output_tokens, attributes)- metrics["genai_completion_tokens"].add(- response.usage.output_tokens, attributes)- metrics["genai_prompt_tokens"].add(- response.usage.input_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.pyindex 7b6bd01c..6cab60fe 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py@@ -1,35 +1,267 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring Anthropic API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_chat_model_cost, handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_messages(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def async_messages(version, environment, application_name, tracer,+ pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for messages to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the OpenAI API.+ application_name: Name of the application using the Anthropic API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of OpenAI usage.+ pricing_info: Information used for calculating the cost of Anthropic usage.trace_content: Flag indicating whether to trace the actual content.Returns:A function that wraps the chat method to add telemetry."""+ class TracedASyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'anthropic.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._input_tokens = ""+ self._output_tokens = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)++ # Collect message IDs and input token from events+ if chunked.get('type') == "message_start":+ self._response_id = chunked.get('message').get('id')+ self._input_tokens = chunked.get('message').get('usage').get('input_tokens')+ self._response_model = chunked.get('message').get('model')+ # Collect message IDs and aggregated response from events+ if chunked.get('type') == "content_block_delta":+ content = chunked.get('delta').get('text')+ if content:+ self._llmresponse += content+ # Collect output tokens and stop reason from events+ if chunked.get('type') == "message_delta":+ self._output_tokens = chunked.get('usage').get('output_tokens')+ self._finish_reason = chunked.get('delta').get('stop_reason')++ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop_sequences", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ self._kwargs.get("top_k", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+async def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'messages' API call to add telemetry.@@ -49,147 +281,27 @@ def async_messages(gen_ai_endpoint, version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.anthropic.com", 443)+ request_model = kwargs.get("model", "claude-3-5-sonnet-latest")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for event in await wrapped(*args, **kwargs):-- # Collect message IDs and input token from events- if event.type == "message_start":- response_id = event.message.id- prompt_tokens = event.message.usage.input_tokens-- # Aggregate response content- if event.type == "content_block_delta":- llmresponse += event.delta.text-- # Collect output tokens and stop reason from events- if event.type == "message_delta":- completion_tokens = event.usage.output_tokens- finish_reason = event.delta.stop_reason- yield event-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(- kwargs.get("model", "claude-3-sonnet-20240229"),- pricing_info, prompt_tokens, completion_tokens- )+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "claude-3-sonnet-20240229"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", ""))- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "claude-3-sonnet-20240229")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- prompt_tokens + completion_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ return TracedASyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()+ response_dict = response_as_dict(response)try:# Format 'messages' into a single stringmessage_prompt = kwargs.get("messages", "")@@ -210,48 +322,71 @@ def async_messages(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)+ input_tokens = response_dict.get('usage').get('input_tokens')+ output_tokens = response_dict.get('usage').get('output_tokens')+# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "claude-3-sonnet-20240229"),- pricing_info, response.usage.input_tokens,- response.usage.output_tokens)+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ llm_response = ""+ for i in range(len(response_dict.get('content'))):+ if response_dict.get('content')[i].get('type') == 'text':+ llm_response = response_dict.get('content')[i].get('text')- # Set Span attribues+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "claude-3-sonnet-20240229"))+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- False)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop_sequences", []))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", ""))+ kwargs.get("top_k", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.stop_reason])+ [response_dict.get('stop_reason')])+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get('id'))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.input_tokens)+ input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.output_tokens)+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)++ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ response_dict.get('content')[0].get('type'))++ # Set Span attributes (Extra)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.input_tokens +- response.usage.output_tokens)+ input_tokens + output_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -263,36 +398,36 @@ def async_messages(gen_ai_endpoint, version, environment, application_name,name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={# pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.content[0].text if response.content else "",+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "claude-3-sonnet-20240229")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- response.usage.input_tokens +- response.usage.output_tokens, attributes)- metrics["genai_completion_tokens"].add(- response.usage.output_tokens, attributes)- metrics["genai_prompt_tokens"].add(- response.usage.input_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.pyindex d5e28b6a..b844a2fe 100644--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py@@ -4,8 +4,6 @@ Module for monitoring OpenAI API calls.import loggingimport time-from urllib.parse import urlparse-from typing import Tuple, Anyfrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (@@ -19,38 +17,13 @@ from openlit.__helpers import (calculate_ttft,calculate_tbt,create_metrics_attributes,+ set_server_address_and_port)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def set_server_address_and_port(client_instance: Any) -> Tuple[str, int]:- """- Determines and returns the server address and port based on the provided client's `base_url`,- using defaults if none found or values are None.- """- default_server_address = "api.openai.com"- default_server_port = 443-- base_client = getattr(client_instance, "_client", None)- base_url = getattr(base_client, "base_url", None)-- if base_url:- if isinstance(base_url, str):- url = urlparse(base_url)- server_address = url.hostname or default_server_address- server_port = url.port if url.port is not None else default_server_port- else: # base_url might not be a str; handle as an object.- server_address = getattr(base_url, "host", None) or default_server_address- port_attr = getattr(base_url, "port", None)- server_port = port_attr if port_attr is not None else default_server_port- else: # no base_url provided; use defaults.- server_address = default_server_address- server_port = default_server_port-- return server_address, server_port-def async_chat_completions(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""@@ -230,6 +203,12 @@ def async_chat_completions(version, environment, application_name,self._openai_response_service_tier)self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,self._openai_system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")# Set Span attributes (Extra)self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,@@ -248,6 +227,8 @@ def async_chat_completions(version, environment, application_name,self._tbt)self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -318,7 +299,7 @@ def async_chat_completions(version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "gpt-4o")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"@@ -333,7 +314,7 @@ def async_chat_completions(version, environment, application_name,# Handling for non-streaming responseselse:- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = await wrapped(*args, **kwargs)end_time = time.time()@@ -423,6 +404,8 @@ def async_chat_completions(version, environment, application_name,cost)span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -446,6 +429,13 @@ def async_chat_completions(version, environment, application_name,span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,str(response_dict.get('choices')[i].get('message').get('tool_calls')))+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")+span.set_status(Status(StatusCode.OK))if disable_metrics is False:@@ -520,7 +510,7 @@ def async_embedding(version, environment, application_name,The response from the original 'embeddings' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "text-embedding-ada-002")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"@@ -569,6 +559,8 @@ def async_embedding(version, environment, application_name,input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -648,7 +640,7 @@ def async_image_generate(version, environment, application_name,The response from the original 'images.generate' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "dall-e-2")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_IMAGE} {request_model}"@@ -709,6 +701,8 @@ def async_image_generate(version, environment, application_name,items.revised_prompt if items.revised_prompt else "")span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,kwargs.get("user", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -795,7 +789,7 @@ def async_image_variatons(version, environment, application_name,The response from the original 'images.create.variations' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "dall-e-2")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_IMAGE} {request_model}"@@ -849,6 +843,8 @@ def async_image_variatons(version, environment, application_name,"standard")span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,kwargs.get("user", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:attribute_name = f"{SemanticConvetion.GEN_AI_RESPONSE_IMAGE}.{images_count}"@@ -929,7 +925,7 @@ def async_audio_create(version, environment, application_name,The response from the original 'audio.speech.create' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "tts-1")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO} {request_model}"@@ -973,6 +969,8 @@ def async_audio_create(version, environment, application_name,kwargs.get("speed", 1))span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.pyindex 105cd7b8..2baaa1c0 100644--- a/sdk/python/src/openlit/instrumentation/openai/openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py@@ -4,8 +4,6 @@ Module for monitoring OpenAI API calls.import loggingimport time-from urllib.parse import urlparse-from typing import Tuple, Anyfrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom openlit.__helpers import (@@ -19,38 +17,13 @@ from openlit.__helpers import (calculate_ttft,calculate_tbt,create_metrics_attributes,+ set_server_address_and_port)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def set_server_address_and_port(client_instance: Any) -> Tuple[str, int]:- """- Determines and returns the server address and port based on the provided client's `base_url`,- using defaults if none found or values are None.- """- default_server_address = "api.openai.com"- default_server_port = 443-- base_client = getattr(client_instance, "_client", None)- base_url = getattr(base_client, "base_url", None)-- if base_url:- if isinstance(base_url, str):- url = urlparse(base_url)- server_address = url.hostname or default_server_address- server_port = url.port if url.port is not None else default_server_port- else: # base_url might not be a str; handle as an object.- server_address = getattr(base_url, "host", None) or default_server_address- port_attr = getattr(base_url, "port", None)- server_port = port_attr if port_attr is not None else default_server_port- else: # no base_url provided; use defaults.- server_address = default_server_address- server_port = default_server_port-- return server_address, server_port-def chat_completions(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""@@ -230,6 +203,12 @@ def chat_completions(version, environment, application_name,self._openai_response_service_tier)self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,self._openai_system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")# Set Span attributes (Extra)self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,@@ -248,6 +227,8 @@ def chat_completions(version, environment, application_name,self._tbt)self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:self._span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -318,7 +299,7 @@ def chat_completions(version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "gpt-4o")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"@@ -423,6 +404,8 @@ def chat_completions(version, environment, application_name,cost)span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -446,6 +429,13 @@ def chat_completions(version, environment, application_name,span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,str(response_dict.get('choices')[i].get('message').get('tool_calls')))+ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")+span.set_status(Status(StatusCode.OK))if disable_metrics is False:@@ -520,7 +510,7 @@ def embedding(version, environment, application_name,The response from the original 'embeddings' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "text-embedding-ada-002")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"@@ -569,6 +559,8 @@ def embedding(version, environment, application_name,input_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -648,7 +640,7 @@ def image_generate(version, environment, application_name,The response from the original 'images.generate' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "dall-e-2")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_IMAGE} {request_model}"@@ -709,6 +701,8 @@ def image_generate(version, environment, application_name,items.revised_prompt if items.revised_prompt else "")span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,kwargs.get("user", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(@@ -795,7 +789,7 @@ def image_variatons(version, environment, application_name,The response from the original 'images.create.variations' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "dall-e-2")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_IMAGE} {request_model}"@@ -849,6 +843,8 @@ def image_variatons(version, environment, application_name,"standard")span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,kwargs.get("user", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:attribute_name = f"{SemanticConvetion.GEN_AI_RESPONSE_IMAGE}.{images_count}"@@ -929,7 +925,7 @@ def audio_create(version, environment, application_name,The response from the original 'audio.speech.create' method."""- server_address, server_port = set_server_address_and_port(instance)+ server_address, server_port = set_server_address_and_port(instance, "api.openai.com", 443)request_model = kwargs.get("model", "tts-1")span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_AUDIO} {request_model}"@@ -973,6 +969,8 @@ def audio_create(version, environment, application_name,kwargs.get("speed", 1))span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 1e8bd696..4e09da3c 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -141,6 +141,7 @@ class SemanticConvetion:GEN_AI_DATA_SOURCES = "gen_ai.data_source_count"GEN_AI_ENVIRONMENT = "gen_ai.environment"GEN_AI_APPLICATION_NAME = "gen_ai.application_name"+ GEN_AI_SDK_VERSION = "gen_ai.sdk.version"# GenAI Response Attributes (Extra)GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"