Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Support Events API in Anthropic Instrumentation for AI content collection (#645)

Patcher01ba7a0265e7
diff --git a/sdk/python/.pylintrc b/sdk/python/.pylintrc
index 91ba1442..2ddac00b 100644
--- a/sdk/python/.pylintrc
+++ b/sdk/python/.pylintrc
@@ -13,7 +13,7 @@ max-positional-arguments=15
max-line-length=120
# Maximum number of instance attributes
-max-attributes=20
+max-attributes=25
[MESSAGES CONTROL]
disable=
@@ -28,4 +28,5 @@ disable=
no-name-in-module,
protected-access,
bare-except,
- expression-not-assigned
\ No newline at end of file
+ expression-not-assigned,
+ unnecessary-direct-lambda-call
\ No newline at end of file
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index e87cf8a4..0319aefc 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.33.11"
+version = "1.33.12"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index e287e8b8..88fdb869 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -21,12 +21,13 @@ def response_as_dict(response):
"""
Return parsed response as a dict
"""
+
# pylint: disable=no-else-return
if isinstance(response, dict):
return response
- if hasattr(response, "model_dump"):
+ if hasattr(response, 'model_dump'):
return response.model_dump()
- elif hasattr(response, "parse"):
+ elif hasattr(response, 'parse'):
return response_as_dict(response.parse())
else:
return response
@@ -34,8 +35,8 @@ def response_as_dict(response):
def get_env_variable(name, arg_value, error_message):
"""
Retrieve an environment variable if the argument is not provided
- and raise an error if both are not set.
"""
+
if arg_value is not None:
return arg_value
value = os.getenv(name)
@@ -47,14 +48,8 @@ def get_env_variable(name, arg_value, error_message):
def openai_tokens(text, model):
"""
Calculate the number of tokens a given text would take up for a specified model.
-
- Args:
- text (str): The input text to be encoded.
- model (str): The model identifier used for encoding.
-
- Returns:
- int: The number of tokens the text is encoded into.
"""
+
try:
encoding = tiktoken.encoding_for_model(model)
except:
@@ -66,16 +61,9 @@ def openai_tokens(text, model):
def general_tokens(text):
"""
Calculate the number of tokens a given text would take up.
-
- Args:
- text (str): The input text to be encoded.
- model (str): The model identifier used for encoding.
-
- Returns:
- int: The number of tokens the text is encoded into.
"""
- encoding = tiktoken.get_encoding("gpt2")
+ encoding = tiktoken.get_encoding('gpt2')
num_tokens = len(encoding.encode(text))
return num_tokens
@@ -83,19 +71,11 @@ def general_tokens(text):
def get_chat_model_cost(model, pricing_info, prompt_tokens, completion_tokens):
"""
Retrieve the cost of processing for a given model based on prompt and tokens.
-
- Args:
- model (str): The model identifier.
- pricing_info (dict): A dictionary containing pricing information for various models.
- prompt_tokens (int): Number of tokens in the prompt.
- completion_tokens (int): Number of tokens in the completion if applicable.
-
- Returns:
- float: The calculated cost for the operation.
"""
+
try:
- cost = ((prompt_tokens / 1000) * pricing_info["chat"][model]["promptPrice"]) + \
- ((completion_tokens / 1000) * pricing_info["chat"][model]["completionPrice"])
+ cost = ((prompt_tokens / 1000) * pricing_info['chat'][model]['promptPrice']) + \
+ ((completion_tokens / 1000) * pricing_info['chat'][model]['completionPrice'])
except:
cost = 0
return cost
@@ -103,17 +83,10 @@ def get_chat_model_cost(model, pricing_info, prompt_tokens, completion_tokens):
def get_embed_model_cost(model, pricing_info, prompt_tokens):
"""
Retrieve the cost of processing for a given model based on prompt tokens.
-
- Args:
- model (str): The model identifier.
- pricing_info (dict): A dictionary containing pricing information for various models.
- prompt_tokens (int): Number of tokens in the prompt.
-
- Returns:
- float: The calculated cost for the operation.
"""
+
try:
- cost = (prompt_tokens / 1000) * pricing_info["embeddings"][model]
+ cost = (prompt_tokens / 1000) * pricing_info['embeddings'][model]
except:
cost = 0
return cost
@@ -121,18 +94,10 @@ def get_embed_model_cost(model, pricing_info, prompt_tokens):
def get_image_model_cost(model, pricing_info, size, quality):
"""
Retrieve the cost of processing for a given model based on image size and quailty.
-
- Args:
- model (str): The model identifier.
- pricing_info (dict): A dictionary containing pricing information for various models.
- size (str): Size of the Image.
- quality (int): Quality of the Image.
-
- Returns:
- float: The calculated cost for the operation.
"""
+
try:
- cost = pricing_info["images"][model][quality][size]
+ cost = pricing_info['images'][model][quality][size]
except:
cost = 0
return cost
@@ -140,20 +105,13 @@ def get_image_model_cost(model, pricing_info, size, quality):
def get_audio_model_cost(model, pricing_info, prompt, duration=None):
"""
Retrieve the cost of processing for a given model based on prompt.
-
- Args:
- model (str): The model identifier.
- pricing_info (dict): A dictionary containing pricing information for various models.
- prompt (str): Prompt to the LLM Model
-
- Returns:
- float: The calculated cost for the operation.
"""
+
try:
if prompt:
- cost = (len(prompt) / 1000) * pricing_info["audio"][model]
+ cost = (len(prompt) / 1000) * pricing_info['audio'][model]
else:
- cost = duration * pricing_info["audio"][model]
+ cost = duration * pricing_info['audio'][model]
except:
cost = 0
return cost
@@ -161,15 +119,10 @@ def get_audio_model_cost(model, pricing_info, prompt, duration=None):
def fetch_pricing_info(pricing_json=None):
"""
Fetches pricing information from a specified URL or File Path.
-
- Args:
- pricing_json(str): path or url to the pricing json file
-
- Returns:
- dict: The pricing json
"""
+
if pricing_json:
- is_url = urlparse(pricing_json).scheme != ""
+ is_url = urlparse(pricing_json).scheme != ''
if is_url:
pricing_url = pricing_json
else:
@@ -177,39 +130,36 @@ def fetch_pricing_info(pricing_json=None):
with open(pricing_json, mode='r', encoding='utf-8') as f:
return json.load(f)
except FileNotFoundError:
- logger.error("Pricing information file not found: %s", pricing_json)
+ logger.error('Pricing information file not found: %s', pricing_json)
except json.JSONDecodeError:
- logger.error("Error decoding JSON from file: %s", pricing_json)
+ logger.error('Error decoding JSON from file: %s', pricing_json)
except Exception as file_err:
- logger.error("Unexpected error occurred while reading file: %s", file_err)
+ logger.error('Unexpected error occurred while reading file: %s', file_err)
return {}
else:
- pricing_url = "https://raw.githubusercontent.com/openlit/openlit/main/assets/pricing.json"
+ pricing_url = 'https://raw.githubusercontent.com/openlit/openlit/main/assets/pricing.json'
try:
# Set a timeout of 10 seconds for both the connection and the read
response = requests.get(pricing_url, timeout=20)
response.raise_for_status()
return response.json()
except requests.HTTPError as http_err:
- logger.error("HTTP error occured while fetching pricing info: %s", http_err)
+ logger.error('HTTP error occured while fetching pricing info: %s', http_err)
except Exception as err:
- logger.error("Unexpected error occurred while fetching pricing info: %s", err)
+ logger.error('Unexpected error occurred while fetching pricing info: %s', err)
return {}
def handle_exception(span,e):
"""Handles Exception when LLM Function fails or trace creation fails."""
- # Record the exception details within the span
+
span.record_exception(e)
span.set_status(Status(StatusCode.ERROR))
def calculate_ttft(timestamps: List[float], start_time: float) -> float:
"""
Calculate the time to the first tokens.
-
- :param timestamps: List of timestamps for received tokens
- :param start_time: The start time of the streaming process
- :return: Time to the first tokens
"""
+
if timestamps:
return timestamps[0] - start_time
return 0.0
@@ -217,10 +167,8 @@ def calculate_ttft(timestamps: List[float], start_time: float) -> float:
def calculate_tbt(timestamps: List[float]) -> float:
"""
Calculate the average time between tokens.
-
- :param timestamps: List of timestamps for received tokens
- :return: Average time between tokens
"""
+
if len(timestamps) > 1:
time_diffs = [timestamps[i] - timestamps[i - 1] for i in range(1, len(timestamps))]
return sum(time_diffs) / len(time_diffs)
@@ -239,8 +187,9 @@ def create_metrics_attributes(
"""
Returns OTel metrics attributes
"""
+
return {
- TELEMETRY_SDK_NAME: "openlit",
+ TELEMETRY_SDK_NAME: 'openlit',
SERVICE_NAME: service_name,
DEPLOYMENT_ENVIRONMENT: deployment_environment,
SemanticConvetion.GEN_AI_OPERATION: operation,
@@ -259,18 +208,18 @@ def set_server_address_and_port(client_instance: Any,
"""
# Try getting base_url from multiple potential attributes
- base_client = getattr(client_instance, "_client", None)
- base_url = getattr(base_client, "base_url", None)
+ base_client = getattr(client_instance, '_client', None)
+ base_url = getattr(base_client, 'base_url', None)
if not base_url:
# Attempt to get endpoint from instance._config.endpoint if base_url is not set
- config = getattr(client_instance, "_config", None)
- base_url = getattr(config, "endpoint", None)
+ config = getattr(client_instance, '_config', None)
+ base_url = getattr(config, 'endpoint', None)
if not base_url:
# Attempt to get server_url from instance.sdk_configuration.server_url
- config = getattr(client_instance, "sdk_configuration", None)
- base_url = getattr(config, "server_url", None)
+ config = getattr(client_instance, 'sdk_configuration', None)
+ base_url = getattr(config, 'server_url', None)
if base_url:
if isinstance(base_url, str):
@@ -278,8 +227,8 @@ def set_server_address_and_port(client_instance: Any,
server_address = url.hostname or default_server_address
server_port = url.port if url.port is not None else default_server_port
else: # base_url might not be a str; handle as an object.
- server_address = getattr(base_url, "host", None) or default_server_address
- port_attr = getattr(base_url, "port", None)
+ server_address = getattr(base_url, 'host', None) or default_server_address
+ port_attr = getattr(base_url, 'port', None)
server_port = port_attr if port_attr is not None else default_server_port
else: # no base_url or endpoint provided; use defaults.
server_address = default_server_address
@@ -301,59 +250,74 @@ def otel_event(name, attributes, body):
def extract_and_format_input(messages):
"""
Process a list of messages to extract content and categorize
- them into fixed roles like 'user', 'assistant', 'system'.
+ them into fixed roles like 'user', 'assistant', 'system', 'tool'.
"""
fixed_roles = ['user', 'assistant', 'system', 'tool'] # Ensure these are your fixed keys
# Initialize the dictionary with fixed keys and empty structures
- formatted_messages = {role_key: {"role": "", "content": ""} for role_key in fixed_roles}
+ formatted_messages = {role_key: {'role': '', 'content': ''} for role_key in fixed_roles}
for message in messages:
# Normalize the message structure
message = response_as_dict(message)
# Extract role and content
- role = message.get("role")
+ role = message.get('role')
if role not in fixed_roles:
continue # Skip any role not in our predefined roles
- content = message.get("content", "")
+ content = message.get('content', '')
# Prepare content as a string
if isinstance(content, list):
content_str = ", ".join(
- # pylint: disable=line-too-long
- f'{item.get("type", "text")}: {item.get("text", item.get("image_url", "").get("url", "") if isinstance(item.get("image_url", ""), dict) else item.get("image_url", ""))}'
+ f'{item.get("type", "text")}: {extract_text_from_item(item)}'
for item in content
)
else:
content_str = content
# Set the role in the formatted message and concatenate content
- if not formatted_messages[role]["role"]:
- formatted_messages[role]["role"] = role
+ if not formatted_messages[role]['role']:
+ formatted_messages[role]['role'] = role
- if formatted_messages[role]["content"]:
- formatted_messages[role]["content"] += " " + content_str
+ if formatted_messages[role]['content']:
+ formatted_messages[role]['content'] += ' ' + content_str
else:
- formatted_messages[role]["content"] = content_str
+ formatted_messages[role]['content'] = content_str
return formatted_messages
+def extract_text_from_item(item):
+ """
+ Extract text from inpit message
+ """
+
+ #pylint: disable=no-else-return
+ if item.get('type') == 'text':
+ return item.get('text', '')
+ elif item.get('type') == 'image':
+ # Handle image content specifically checking for 'url' or 'base64'
+ source = item.get('source', {})
+ if isinstance(source, dict):
+ if source.get('type') == 'base64':
+ # Return the actual base64 data if present
+ return source.get('data', '[Missing base64 data]')
+ elif source.get('type') == 'url':
+ return source.get('url', '[Missing URL]')
+ elif item.get('type') == 'image_url':
+ # New format: Handle the 'image_url' type
+ image_url = item.get('image_url', {})
+ if isinstance(image_url, dict):
+ return image_url.get('url', '[Missing image URL]')
+ return ''
+
# To be removed one the change to log events (from span events) is complete
def concatenate_all_contents(formatted_messages):
"""
- Concatenate all 'content' fields from the formatted messages
- dictionary into a single string.
-
- Parameters:
- - formatted_messages: Dictionary with roles as keys and corresponding
- role and content as values.
-
- Returns:
- - A single string with all content concatenated.
+ Concatenate all 'content' fields into a single strin
"""
- return " ".join(
+ return ' '.join(
message_data['content']
for message_data in formatted_messages.values()
if message_data['content']
diff --git a/sdk/python/src/openlit/instrumentation/ag2/__init__.py b/sdk/python/src/openlit/instrumentation/ag2/__init__.py
index 7362213e..b6dbccc0 100644
--- a/sdk/python/src/openlit/instrumentation/ag2/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/ag2/__init__.py
@@ -9,7 +9,7 @@ from openlit.instrumentation.ag2.ag2 import (
conversable_agent, agent_run
)
-_instruments = ("ag2 >= 0.3.2",)
+_instruments = ('ag2 >= 0.3.2',)
class AG2Instrumentor(BaseInstrumentor):
"""
@@ -20,26 +20,26 @@ class AG2Instrumentor(BaseInstrumentor):
return _instruments
def _instrument(self, **kwargs):
- application_name = kwargs.get("application_name", "default_application")
- environment = kwargs.get("environment", "default_environment")
- tracer = kwargs.get("tracer")
- event_provider = kwargs.get("event_provider")
- metrics = kwargs.get("metrics_dict")
- pricing_info = kwargs.get("pricing_info", {})
- capture_message_content = kwargs.get("capture_message_content", False)
- disable_metrics = kwargs.get("disable_metrics")
- version = importlib.metadata.version("ag2")
+ application_name = kwargs.get('application_name', 'default_application')
+ environment = kwargs.get('environment', 'default_environment')
+ tracer = kwargs.get('tracer')
+ event_provider = kwargs.get('event_provider')
+ metrics = kwargs.get('metrics_dict')
+ pricing_info = kwargs.get('pricing_info', {})
+ capture_message_content = kwargs.get('capture_message_content', False)
+ disable_metrics = kwargs.get('disable_metrics')
+ version = importlib.metadata.version('ag2')
wrap_function_wrapper(
- "autogen.agentchat.conversable_agent",
- "ConversableAgent.__init__",
+ 'autogen.agentchat.conversable_agent',
+ 'ConversableAgent.__init__',
conversable_agent(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
wrap_function_wrapper(
- "autogen.agentchat.conversable_agent",
- "ConversableAgent.run",
+ 'autogen.agentchat.conversable_agent',
+ 'ConversableAgent.run',
agent_run(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
diff --git a/sdk/python/src/openlit/instrumentation/ag2/ag2.py b/sdk/python/src/openlit/instrumentation/ag2/ag2.py
index 53834635..1dd305a0 100644
--- a/sdk/python/src/openlit/instrumentation/ag2/ag2.py
+++ b/sdk/python/src/openlit/instrumentation/ag2/ag2.py
@@ -28,7 +28,7 @@ def set_span_attributes(span, version, operation_name, environment,
"""
# Set Span attributes (OTel Semconv)
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION, operation_name)
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM, SemanticConvetion.GEN_AI_SYSTEM_AG2)
span.set_attribute(SemanticConvetion.GEN_AI_AGENT_NAME, AGENT_NAME)
@@ -73,10 +73,10 @@ def emit_events(response, event_provider, capture_message_content):
SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_AG2
},
body={
- "index": response.chat_history.index(chat),
- "message": {
- **({"content": chat['content']} if capture_message_content else {}),
- "role": 'assistant' if chat['role'] == 'user' else 'user'
+ 'index': response.chat_history.index(chat),
+ 'message': {
+ **({'content': chat['content']} if capture_message_content else {}),
+ 'role': 'assistant' if chat['role'] == 'user' else 'user'
}
}
)
@@ -92,12 +92,12 @@ def conversable_agent(version, environment, application_name,
global AGENT_NAME, MODEL_AND_NAME_SET, REQUEST_MODEL, SYSTEM_MESSAGE
if not MODEL_AND_NAME_SET:
- AGENT_NAME = kwargs.get("name", "NOT_FOUND")
- REQUEST_MODEL = kwargs.get("llm_config", {}).get('model', 'gpt-4o')
+ AGENT_NAME = kwargs.get('name', 'NOT_FOUND')
+ REQUEST_MODEL = kwargs.get('llm_config', {}).get('model', 'gpt-4o')
SYSTEM_MESSAGE = kwargs.get('system_message', '')
MODEL_AND_NAME_SET = True
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CREATE_AGENT} {AGENT_NAME}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CREATE_AGENT} {AGENT_NAME}'
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
try:
@@ -117,7 +117,7 @@ def conversable_agent(version, environment, application_name,
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
return response
return wrapper
@@ -130,7 +130,7 @@ def agent_run(version, environment, application_name,
def wrapper(wrapped, instance, args, kwargs):
server_address, server_port = '127.0.0.1', 80
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EXECUTE_AGENT_TASK} {AGENT_NAME}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_EXECUTE_AGENT_TASK} {AGENT_NAME}'
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
try:
@@ -157,7 +157,7 @@ def agent_run(version, environment, application_name,
except Exception as e:
handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
return response
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/ai21/__init__.py b/sdk/python/src/openlit/instrumentation/ai21/__init__.py
index 9606edc5..1885fd09 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/__init__.py
@@ -13,7 +13,7 @@ from openlit.instrumentation.ai21.async_ai21 import (
async_chat, async_chat_rag
)
-_instruments = ("ai21 >= 3.0.0",)
+_instruments = ('ai21 >= 3.0.0',)
class AI21Instrumentor(BaseInstrumentor):
"""
@@ -24,40 +24,40 @@ class AI21Instrumentor(BaseInstrumentor):
return _instruments
def _instrument(self, **kwargs):
- application_name = kwargs.get("application_name", "default_application")
- environment = kwargs.get("environment", "default_environment")
- tracer = kwargs.get("tracer")
- event_provider = kwargs.get("event_provider")
- metrics = kwargs.get("metrics_dict")
- pricing_info = kwargs.get("pricing_info", {})
- capture_message_content = kwargs.get("capture_message_content", False)
- disable_metrics = kwargs.get("disable_metrics")
- version = importlib.metadata.version("ai21")
+ application_name = kwargs.get('application_name', 'default')
+ environment = kwargs.get('environment', 'default')
+ tracer = kwargs.get('tracer')
+ event_provider = kwargs.get('event_provider')
+ metrics = kwargs.get('metrics_dict')
+ pricing_info = kwargs.get('pricing_info', {})
+ capture_message_content = kwargs.get('capture_message_content', False)
+ disable_metrics = kwargs.get('disable_metrics')
+ version = importlib.metadata.version('ai21')
#sync
wrap_function_wrapper(
- "ai21.clients.studio.resources.chat.chat_completions",
- "ChatCompletions.create",
+ 'ai21.clients.studio.resources.chat.chat_completions',
+ 'ChatCompletions.create',
chat(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
wrap_function_wrapper(
- "ai21.clients.studio.resources.studio_conversational_rag",
- "StudioConversationalRag.create",
+ 'ai21.clients.studio.resources.studio_conversational_rag',
+ 'StudioConversationalRag.create',
chat_rag(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
#Async
wrap_function_wrapper(
- "ai21.clients.studio.resources.chat.async_chat_completions",
- "AsyncChatCompletions.create",
+ 'ai21.clients.studio.resources.chat.async_chat_completions',
+ 'AsyncChatCompletions.create',
async_chat(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
wrap_function_wrapper(
- "ai21.clients.studio.resources.studio_conversational_rag",
- "AsyncStudioConversationalRag.create",
+ 'ai21.clients.studio.resources.studio_conversational_rag',
+ 'AsyncStudioConversationalRag.create',
async_chat_rag(version, environment, application_name,
tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
diff --git a/sdk/python/src/openlit/instrumentation/ai21/ai21.py b/sdk/python/src/openlit/instrumentation/ai21/ai21.py
index 16b7642f..d6ecf756 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/ai21.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/ai21.py
@@ -7,7 +7,6 @@ import time
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
handle_exception,
- response_as_dict,
set_server_address_and_port,
)
from openlit.instrumentation.ai21.utils import (
@@ -47,9 +46,9 @@ def chat(version, environment, application_name,
self._span = span
self._span_name = span_name
# Placeholder for aggregating streaming response
- self._llmresponse = ""
- self._response_id = ""
- self._finish_reason = ""
+ self._llmresponse = ''
+ self._response_id = ''
+ self._finish_reason = ''
self._input_tokens = 0
self._output_tokens = 0
self._choices = []
@@ -100,7 +99,7 @@ def chat(version, environment, application_name,
)
except Exception as e:
handle_exception(self._span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
raise
def wrapper(wrapped, instance, args, kwargs):
@@ -109,12 +108,12 @@ def chat(version, environment, application_name,
"""
# Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
+ streaming = kwargs.get('stream', False)
- server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
- request_model = kwargs.get("model", "jamba-1.5-mini")
+ server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
+ request_model = kwargs.get('model', 'jamba-1.5-mini')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
# pylint: disable=no-else-return
if streaming:
@@ -129,7 +128,7 @@ def chat(version, environment, application_name,
start_time = time.time()
response = wrapped(*args, **kwargs)
response = process_chat_response(
- response=response_as_dict(response),
+ response=response,
request_model=request_model,
pricing_info=pricing_info,
server_port=server_port,
@@ -161,16 +160,16 @@ def chat_rag(version, environment, application_name,
Wraps the GenAI function call.
"""
- server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
- request_model = kwargs.get("model", "jamba-1.5-mini")
+ server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
+ request_model = kwargs.get('model', 'jamba-1.5-mini')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = wrapped(*args, **kwargs)
response = process_chat_rag_response(
- response=response_as_dict(response),
+ response=response,
request_model=request_model,
pricing_info=pricing_info,
server_port=server_port,
diff --git a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
index 11359aa8..b04b798a 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
@@ -7,7 +7,6 @@ import time
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
handle_exception,
- response_as_dict,
set_server_address_and_port,
)
from openlit.instrumentation.ai21.utils import (
@@ -47,9 +46,9 @@ def async_chat(version, environment, application_name,
self._span = span
self._span_name = span_name
# Placeholder for aggregating streaming response
- self._llmresponse = ""
- self._response_id = ""
- self._finish_reason = ""
+ self._llmresponse = ''
+ self._response_id = ''
+ self._finish_reason = ''
self._input_tokens = 0
self._output_tokens = 0
self._choices = []
@@ -100,7 +99,7 @@ def async_chat(version, environment, application_name,
)
except Exception as e:
handle_exception(self._span, e)
- logger.error("Error in trace creation: %s", e)
+ logger.error('Error in trace creation: %s', e)
raise
async def wrapper(wrapped, instance, args, kwargs):
@@ -109,12 +108,12 @@ def async_chat(version, environment, application_name,
"""
# Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
+ streaming = kwargs.get('stream', False)
- server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
- request_model = kwargs.get("model", "jamba-1.5-mini")
+ server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
+ request_model = kwargs.get('model', 'jamba-1.5-mini')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
# pylint: disable=no-else-return
if streaming:
@@ -129,7 +128,7 @@ def async_chat(version, environment, application_name,
start_time = time.time()
response = await wrapped(*args, **kwargs)
response = process_chat_response(
- response=response_as_dict(response),
+ response=response,
request_model=request_model,
pricing_info=pricing_info,
server_port=server_port,
@@ -161,16 +160,16 @@ def async_chat_rag(version, environment, application_name,
Wraps the GenAI function call.
"""
- server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
- request_model = kwargs.get("model", "jamba-1.5-mini")
+ server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
+ request_model = kwargs.get('model', 'jamba-1.5-mini')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = await wrapped(*args, **kwargs)
response = process_chat_rag_response(
- response=response_as_dict(response),
+ response=response,
request_model=request_model,
pricing_info=pricing_info,
server_port=server_port,
diff --git a/sdk/python/src/openlit/instrumentation/ai21/utils.py b/sdk/python/src/openlit/instrumentation/ai21/utils.py
index 35cb9d24..04ed51d4 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/utils.py
@@ -29,30 +29,30 @@ def setup_common_span_attributes(span, request_model, kwargs, tokens,
"""
# Base attributes from SDK and operation settings.
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
span.set_attribute(SemanticConvetion.GEN_AI_OPERATION, SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM, SemanticConvetion.GEN_AI_SYSTEM_AI21)
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL, request_model)
span.set_attribute(SemanticConvetion.SERVER_PORT, server_port)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED, kwargs.get("seed", ""))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, kwargs.get("frequency_penalty", 0.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, kwargs.get("presence_penalty", 0.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, kwargs.get("stop", []))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, kwargs.get("temperature", 0.4))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P, kwargs.get("top_p", 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED, kwargs.get('seed', ''))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY, kwargs.get('frequency_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, kwargs.get('max_tokens', -1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY, kwargs.get('presence_penalty', 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, kwargs.get('stop', []))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, kwargs.get('temperature', 0.4))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P, kwargs.get('top_p', 1.0))
# Add token-related attributes if available.
- if "finish_reason" in tokens:
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON, [tokens["finish_reason"]])
- if "response_id" in tokens:
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID, tokens["response_id"])
- if "input_tokens" in tokens:
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS, tokens["input_tokens"])
- if "output_tokens" in tokens:
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS, tokens["output_tokens"])
- if "total_tokens" in tokens:
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS, tokens["total_tokens"])
+ if 'finish_reason' in tokens:
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON, [tokens['finish_reason']])
+ if 'response_id' in tokens:
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID, tokens['response_id'])
+ if 'input_tokens' in tokens:
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS, tokens['input_tokens'])
+ if 'output_tokens' in tokens:
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS, tokens['output_tokens'])
+ if 'total_tokens' in tokens:
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS, tokens['total_tokens'])
span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL, request_model)
span.set_attribute(SemanticConvetion.SERVER_ADDRESS, server_address)
@@ -80,15 +80,15 @@ def record_common_metrics(metrics, application_name, environment, request_model,
server_port=server_port,
response_model=request_model,
)
- metrics["genai_client_usage_tokens"].record(input_tokens + output_tokens, attributes)
- metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)
+ metrics['genai_client_usage_tokens'].record(input_tokens + output_tokens, attributes)
+ metrics['genai_client_operation_duration'].record(end_time - start_time, attributes)
if include_tbt and tbt_value is not None:
- metrics["genai_server_tbt"].record(tbt_value, attributes)
- metrics["genai_server_ttft"].record(end_time - start_time, attributes)
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_completion_tokens"].add(output_tokens, attributes)
- metrics["genai_prompt_tokens"].add(input_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
+ metrics['genai_server_tbt'].record(tbt_value, attributes)
+ metrics['genai_server_ttft'].record(end_time - start_time, attributes)
+ metrics['genai_requests'].add(1, attributes)
+ metrics['genai_completion_tokens'].add(output_tokens, attributes)
+ metrics['genai_prompt_tokens'].add(input_tokens, attributes)
+ metrics['genai_cost'].record(cost, attributes)
def emit_common_events(event_provider, choices, finish_reason, llmresponse, formatted_messages,
capture_message_content, n):
@@ -99,25 +99,25 @@ def emit_common_events(event_provider, choices, finish_reason, llmresponse, form
if n > 1:
for choice in choices:
choice_event_body = {
- "finish_reason": finish_reason,
- "index": choice.get('index', 0),
- "message": {
- **({"content": choice.get('message', {}).get('content', '')} if capture_message_content else {}),
- "role": choice.get('message', {}).get('role', 'assistant')
+ 'finish_reason': finish_reason,
+ 'index': choice.get('index', 0),
+ 'message': {
+ **({'content': choice.get('message', {}).get('content', '')} if capture_message_content else {}),
+ 'role': choice.get('message', {}).get('role', 'assistant')
}
}
# If tool calls exist, emit an event for each tool call.
tool_calls = choice.get('message', {}).get('tool_calls')
if tool_calls:
for tool_call in tool_calls:
- choice_event_body["message"].update({
- "tool_calls": {
- "function": {
- "name": tool_call.get('function', {}).get('name', ''),
- "arguments": tool_call.get('function', {}).get('arguments', '')
+ choice_event_body['message'].update({
+ 'tool_calls': {
+ 'function': {
+ 'name': tool_call.get('function', {}).get('name', ''),
+ 'arguments': tool_call.get('function', {}).get('arguments', '')
},
- "id": tool_call.get('id', ''),
- "type": tool_call.get('type', 'function')
+ 'id': tool_call.get('id', ''),
+ 'type': tool_call.get('type', 'function')
}
})
event = otel_event(
@@ -136,11 +136,11 @@ def emit_common_events(event_provider, choices, finish_reason, llmresponse, form
else:
# Single choice case.
choice_event_body = {
- "finish_reason": finish_reason,
- "index": 0,
- "message": {
- **({"content": llmresponse} if capture_message_content else {}),
- "role": 'assistant'
+ 'finish_reason': finish_reason,
+ 'index': 0,
+ 'message': {
+ **({'content': llmresponse} if capture_message_content else {}),
+ 'role': 'assistant'
}
}
event = otel_event(
@@ -155,25 +155,25 @@ def emit_common_events(event_provider, choices, finish_reason, llmresponse, form
msg = formatted_messages.get(role, {})
if msg.get('content', ''):
event_body = {
- **({"content": msg.get('content', '')} if capture_message_content else {}),
- "role": msg.get('role', [])
+ **({'content': msg.get('content', '')} if capture_message_content else {}),
+ 'role': msg.get('role', [])
}
# For assistant messages, attach tool call details if they exist.
if role == 'assistant' and choices:
tool_calls = choices[0].get('message', {}).get('tool_calls', [])
if tool_calls:
- event_body["tool_calls"] = {
- "function": {
- "name": tool_calls[0].get('function', {}).get('name', ''),
- "arguments": tool_calls[0].get('function', {}).get('arguments', '')
+ event_body['tool_calls'] = {
+ 'function': {
+ 'name': tool_calls[0].get('function', {}).get('name', ''),
+ 'arguments': tool_calls[0].get('function', {}).get('arguments', '')
},
- "id": tool_calls[0].get('id', ''),
- "type": "function"
+ 'id': tool_calls[0].get('id', ''),
+ 'type': 'function'
}
if role == 'tool' and choices:
tool_calls = choices[0].get('message', {}).get('tool_calls', [])
if tool_calls:
- event_body["id"] = tool_calls[0].get('id', '')
+ event_body['id'] = tool_calls[0].get('id', '')
event = otel_event(
name=getattr(SemanticConvetion, f'GEN_AI_{role.upper()}_MESSAGE'),
attributes={SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_AI21},
@@ -201,8 +201,8 @@ def process_chunk(self, chunk):
if content:
self._llmresponse += content
if chunked.get('usage'):
- self._input_tokens = chunked.get('usage').get("prompt_tokens")
- self._output_tokens = chunked.get('usage').get("completion_tokens")
+ self._input_tokens = chunked.get('usage').get('prompt_tokens')
+ self._output_tokens = chunked.get('usage').get('completion_tokens')
self._response_id = chunked.get('id')
self._choices += chunked.get('choices')
self._finish_reason = chunked.get('choices')[0].get('finish_reason')
@@ -218,19 +218,19 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
scope._tbt = calculate_tbt(scope._timestamps)
# Extract and format input messages.
- formatted_messages = extract_and_format_input(scope._kwargs.get("messages", ""))
+ formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))
prompt = concatenate_all_contents(formatted_messages)
- request_model = scope._kwargs.get("model", "jamba-1.5-mini")
+ request_model = scope._kwargs.get('model', 'jamba-1.5-mini')
# Calculate cost based on token usage.
cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)
# Prepare tokens dictionary.
tokens = {
- "finish_reason": scope._finish_reason,
- "response_id": scope._response_id,
- "input_tokens": scope._input_tokens,
- "output_tokens": scope._output_tokens,
- "total_tokens": scope._input_tokens + scope._output_tokens,
+ 'finish_reason': scope._finish_reason,
+ 'response_id': scope._response_id,
+ 'input_tokens': scope._input_tokens,
+ 'output_tokens': scope._output_tokens,
+ 'total_tokens': scope._input_tokens + scope._output_tokens,
}
extra_attrs = {
SemanticConvetion.GEN_AI_REQUEST_IS_STREAM: is_stream,
@@ -239,7 +239,7 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
SemanticConvetion.GEN_AI_SERVER_TBT: scope._tbt,
SemanticConvetion.GEN_AI_SERVER_TTFT: scope._ttft,
SemanticConvetion.GEN_AI_SDK_VERSION: version,
- SemanticConvetion.GEN_AI_OUTPUT_TYPE: "text" if isinstance(scope._llmresponse, str) else "json"
+ SemanticConvetion.GEN_AI_OUTPUT_TYPE: 'text' if isinstance(scope._llmresponse, str) else 'json'
}
# Set span attributes.
setup_common_span_attributes(scope._span, request_model, scope._kwargs, tokens,
@@ -282,13 +282,14 @@ def process_streaming_chat_response(self, pricing_info, environment, application
def process_chat_response(response, request_model, pricing_info, server_port, server_address,
environment, application_name, metrics, event_provider, start_time,
- span, capture_message_content=False, disable_metrics=False, version="1.0.0", **kwargs):
+ span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
"""
Process a synchronous chat response and generate Telemetry.
"""
# Create a generic scope object to hold telemetry data.
self = type('GenericScope', (), {})()
+ response_dict = response_as_dict(response)
# pylint: disable = no-member
self._start_time = start_time
@@ -298,19 +299,19 @@ def process_chat_response(response, request_model, pricing_info, server_port, se
# Concatenate content from all choices.
self._llmresponse = ''.join(
(choice.get('message', {}).get('content') or '')
- for choice in response.get('choices', [])
+ for choice in response_dict.get('choices', [])
)
- self._response_role = response.get('message', {}).get('role', 'assistant')
- self._input_tokens = response.get('usage', {}).get('prompt_tokens', 0)
- self._output_tokens = response.get('usage', {}).get('completion_tokens', 0)
- self._response_id = response.get('id', '')
+ self._response_role = response_dict.get('message', {}).get('role', 'assistant')
+ self._input_tokens = response_dict.get('usage', {}).get('prompt_tokens', 0)
+ self._output_tokens = response_dict.get('usage', {}).get('completion_tokens', 0)
+ self._response_id = response_dict.get('id', '')
self._response_model = request_model
- self._finish_reason = response.get('choices', [{}])[0].get('finish_reason')
+ self._finish_reason = response_dict.get('choices', [{}])[0].get('finish_reason')
self._timestamps = []
self._ttft, self._tbt = self._end_time - self._start_time, 0
self._server_address, self._server_port = server_address, server_port
self._kwargs = kwargs
- self._choices = response.get('choices')
+ self._choices = response_dict.get('choices')
common_chat_logic(self, pricing_info, environment, application_name, metrics,
event_provider, capture_message_content, disable_metrics, version, is_stream=False)
@@ -319,30 +320,31 @@ def process_chat_response(response, request_model, pricing_info, server_port, se
def process_chat_rag_response(response, request_model, pricing_info, server_port, server_address,
environment, application_name, metrics, event_provider, start_time,
- span, capture_message_content=False, disable_metrics=False, version="1.0.0", **kwargs):
+ span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
"""
Process a chat response and generate Telemetry.
"""
end_time = time.time()
+ response_dict = response_as_dict(response)
try:
# Format input messages into a single prompt string.
- messages_input = kwargs.get("messages", "")
+ messages_input = kwargs.get('messages', '')
formatted_messages = extract_and_format_input(messages_input)
prompt = concatenate_all_contents(formatted_messages)
input_tokens = general_tokens(prompt)
# Create tokens dict and RAG-specific extra attributes.
- tokens = {"response_id": response.get("id"), "input_tokens": input_tokens}
+ tokens = {'response_id': response_dict.get('id'), 'input_tokens': input_tokens}
extra_attrs = {
SemanticConvetion.GEN_AI_REQUEST_IS_STREAM: False,
SemanticConvetion.GEN_AI_SERVER_TTFT: end_time - start_time,
SemanticConvetion.GEN_AI_SDK_VERSION: version,
- SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS: kwargs.get("max_segments", -1),
- SemanticConvetion.GEN_AI_RAG_STRATEGY: kwargs.get("retrieval_strategy", "segments"),
- SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD: kwargs.get("retrieval_similarity_threshold", -1),
- SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS: kwargs.get("max_neighbors", -1),
- SemanticConvetion.GEN_AI_RAG_FILE_IDS: str(kwargs.get("file_ids", "")),
- SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH: kwargs.get("path", "")
+ SemanticConvetion.GEN_AI_RAG_MAX_SEGMENTS: kwargs.get('max_segments', -1),
+ SemanticConvetion.GEN_AI_RAG_STRATEGY: kwargs.get('retrieval_strategy', 'segments'),
+ SemanticConvetion.GEN_AI_RAG_SIMILARITY_THRESHOLD: kwargs.get('retrieval_similarity_threshold', -1),
+ SemanticConvetion.GEN_AI_RAG_MAX_NEIGHBORS: kwargs.get('max_neighbors', -1),
+ SemanticConvetion.GEN_AI_RAG_FILE_IDS: str(kwargs.get('file_ids', '')),
+ SemanticConvetion.GEN_AI_RAG_DOCUMENTS_PATH: kwargs.get('path', '')
}
# Set common span attributes.
setup_common_span_attributes(span, request_model, kwargs, tokens,
@@ -357,7 +359,7 @@ def process_chat_rag_response(response, request_model, pricing_info, server_port
)
output_tokens = 0
- choices = response.get('choices', [])
+ choices = response_dict.get('choices', [])
# Instead of adding a separate event per choice, we aggregate all completion content.
aggregated_completion = []
for i in range(kwargs.get('n', 1)):
@@ -370,14 +372,14 @@ def process_chat_rag_response(response, request_model, pricing_info, server_port
str(choices[i].get('message', {}).get('tool_calls')))
# Set output type based on actual content type.
if isinstance(content, str):
- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE, "text")
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE, 'text')
elif content is not None:
- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE, "json")
+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE, 'json')
# Concatenate completion responses.
llmresponse = ''.join(aggregated_completion)
- tokens["output_tokens"] = output_tokens
- tokens["total_tokens"] = input_tokens + output_tokens
+ tokens['output_tokens'] = output_tokens
+ tokens['total_tokens'] = input_tokens + output_tokens
cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST, cost)
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/__init__.py b/sdk/python/src/openlit/instrumentation/anthropic/__init__.py
index dd3b7091..ea14e074 100644
--- a/sdk/python/src/openlit/instrumentation/anthropic/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/anthropic/__init__.py
@@ -1,4 +1,3 @@
-# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
"""Initializer of Auto Instrumentation of Anthropic Functions"""
from typing import Collection
@@ -9,7 +8,7 @@ from wrapt import wrap_function_wrapper
from openlit.instrumentation.anthropic.anthropic import messages
from openlit.instrumentation.anthropic.async_anthropic import async_messages
-_instruments = ("anthropic >= 0.21.0",)
+_instruments = ('anthropic >= 0.21.0',)
class AnthropicInstrumentor(BaseInstrumentor):
"""
@@ -20,29 +19,30 @@ class AnthropicInstrumentor(BaseInstrumentor):
return _instruments
def _instrument(self, **kwargs):
- application_name = kwargs.get("application_name", "default")
- environment = kwargs.get("environment", "default")
- tracer = kwargs.get("tracer")
- metrics = kwargs.get("metrics_dict")
- pricing_info = kwargs.get("pricing_info", {})
- capture_message_content = kwargs.get("capture_message_content", False)
- disable_metrics = kwargs.get("disable_metrics")
- version = importlib.metadata.version("anthropic")
+ application_name = kwargs.get('application_name', 'default')
+ environment = kwargs.get('environment', 'default')
+ tracer = kwargs.get('tracer')
+ event_provider = kwargs.get('event_provider')
+ metrics = kwargs.get('metrics_dict')
+ pricing_info = kwargs.get('pricing_info', {})
+ capture_message_content = kwargs.get('capture_message_content', False)
+ disable_metrics = kwargs.get('disable_metrics')
+ version = importlib.metadata.version('anthropic')
#sync
wrap_function_wrapper(
- "anthropic.resources.messages",
- "Messages.create",
+ 'anthropic.resources.messages',
+ 'Messages.create',
messages(version, environment, application_name,
- tracer, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
#async
wrap_function_wrapper(
- "anthropic.resources.messages",
- "AsyncMessages.create",
+ 'anthropic.resources.messages',
+ 'AsyncMessages.create',
async_messages(version, environment, application_name,
- tracer, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
)
def _uninstrument(self, **kwargs):
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py
index ebdfbf1c..0d1eb91f 100644
--- a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py
+++ b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py
@@ -4,54 +4,37 @@ Module for monitoring Anthropic API calls.
import logging
import time
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.trace import SpanKind
from openlit.__helpers import (
- get_chat_model_cost,
handle_exception,
- response_as_dict,
- calculate_ttft,
- calculate_tbt,
- create_metrics_attributes,
set_server_address_and_port
)
+from openlit.instrumentation.anthropic.utils import (
+ process_chunk,
+ process_chat_response,
+ process_streaming_chat_response,
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def messages(version, environment, application_name, tracer,
+def messages(version, environment, application_name, tracer, event_provider,
pricing_info, capture_message_content, metrics, disable_metrics):
"""
- Generates a telemetry wrapper for messages to collect metrics.
-
- Args:
- version: Version of the monitoring package.
- environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Anthropic API.
- tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of Anthropic usage.
- capture_message_content: Flag indicating whether to trace the actual content.
-
- Returns:
- A function that wraps the chat method to add telemetry.
+ Generates a telemetry wrapper for GenAI function call
"""
class TracedSyncStream:
"""
- Wrapper for streaming responses to collect metrics and trace data.
- Wraps the response to collect message IDs and aggregated response.
-
- This class implements the '__aiter__' and '__anext__' methods that
- handle asynchronous streaming responses.
-
- This class also implements '__aenter__' and '__aexit__' methods that
- handle asynchronous context management protocol.
+ Wrapper for streaming responses to collect telemetry.
"""
+
def __init__(
self,
wrapped,
span,
+ span_name,
kwargs,
server_address,
server_port,
@@ -59,13 +42,18 @@ def messages(version, environment, application_name, tracer,
):
self.__wrapped__ = wrapped
self._span = span
- # Placeholder for aggregating streaming response
- self._llmresponse = ""
- self._response_id = ""
- self._response_model = ""
- self._finish_reason = ""
- self._input_tokens = ""
- self._output_tokens = ""
+ self._span_name = span_name
+ self._llmresponse = ''
+ self._response_id = ''
+ self._response_model = ''
+ self._finish_reason = ''
+ self._input_tokens = ''
+ self._output_tokens = ''
+ self._tool_arguments = ''
+ self._tool_id = ''
+ self._tool_name = ''
+ self._tool_calls = None
+ self._response_role = ''
self._args = args
self._kwargs = kwargs
@@ -94,348 +82,68 @@ def messages(version, environment, application_name, tracer,
def __next__(self):
try:
chunk = self.__wrapped__.__next__()
- end_time = time.time()
- # Record the timestamp for the current chunk
- self._timestamps.append(end_time)
-
- if len(self._timestamps) == 1:
- # Calculate time to first chunk
- self._ttft = calculate_ttft(self._timestamps, self._start_time)
-
- chunked = response_as_dict(chunk)
-
- # Collect message IDs and input token from events
- if chunked.get('type') == "message_start":
- self._response_id = chunked.get('message').get('id')
- self._input_tokens = chunked.get('message').get('usage').get('input_tokens')
- self._response_model = chunked.get('message').get('model')
- # Collect message IDs and aggregated response from events
- if chunked.get('type') == "content_block_delta":
- content = chunked.get('delta').get('text')
- if content:
- self._llmresponse += content
- # Collect output tokens and stop reason from events
- if chunked.get('type') == "message_delta":
- self._output_tokens = chunked.get('usage').get('output_tokens')
- self._finish_reason = chunked.get('delta').get('stop_reason')
-
+ process_chunk(self, chunk)
return chunk
except StopIteration:
- # Handling exception ensure observability without disrupting operation
try:
- self._end_time = time.time()
- if len(self._timestamps) > 1:
- self._tbt = calculate_tbt(self._timestamps)
-
- # Format 'messages' into a single string
- message_prompt = self._kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str_list = []
- for item in content:
- if item["type"] == "text":
- content_str_list.append(f'text: {item["text"]}')
- elif (item["type"] == "image_url" and
- not item["image_url"]["url"].startswith("data:")):
- content_str_list.append(f'image_url: {item["image_url"]["url"]}')
- content_str = ", ".join(content_str_list)
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- request_model = self._kwargs.get("model", "claude-3-5-sonnet-latest")
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(request_model,
- pricing_info, self._input_tokens,
- self._output_tokens)
-
- # Set Span attributes (OTel Semconv)
- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- request_model)
- self._span.set_attribute(SemanticConvetion.SERVER_PORT,
- self._server_port)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- self._kwargs.get("max_tokens", -1))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
- self._kwargs.get("stop_sequences", []))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- self._kwargs.get("temperature", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- self._kwargs.get("top_k", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- self._kwargs.get("top_p", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [self._finish_reason])
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- self._response_id)
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
- self._response_model)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- self._input_tokens)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- self._output_tokens)
- self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
- self._server_address)
- if isinstance(self._llmresponse, str):
- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- "text")
- else:
- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- "json")
-
- # Set Span attributes (Extra)
- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- self._span.set_attribute(SERVICE_NAME,
- application_name)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- self._input_tokens + self._output_tokens)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
- self._tbt)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
- self._ttft)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
- version)
- if capture_message_content:
- self._span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- self._span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
- },
+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ event_provider=event_provider,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version
)
- self._span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = create_metrics_attributes(
- service_name=application_name,
- deployment_environment=environment,
- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
- request_model=request_model,
- server_address=self._server_address,
- server_port=self._server_port,
- response_model=self._response_model,
- )
-
- metrics["genai_client_usage_tokens"].record(
- self._input_tokens + self._output_tokens, attributes
- )
- metrics["genai_client_operation_duration"].record(
- self._end_time - self._start_time, attributes
- )
- metrics["genai_server_tbt"].record(
- self._tbt, attributes
- )
- metrics["genai_server_ttft"].record(
- self._ttft, attributes
- )
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)
- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
except Exception as e:
handle_exception(self._span, e)
logger.error("Error in trace creation: %s", e)
- finally:
- self._span.end()
raise
def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'messages' API call to add telemetry.
-
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
-
- Args:
- wrapped: The original 'messages' method to be wrapped.
- instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'messages' method.
- kwargs: Keyword arguments for the 'messages' method.
-
- Returns:
- The response from the original 'messages' method.
+ Wraps the GenAI function call.
"""
- # Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(instance, "api.anthropic.com", 443)
- request_model = kwargs.get("model", "claude-3-5-sonnet-latest")
+ streaming = kwargs.get('stream', False)
+ server_address, server_port = set_server_address_and_port(instance, 'api.anthropic.com', 443)
+ request_model = kwargs.get('model', 'claude-3-5-sonnet-latest')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
# pylint: disable=no-else-return
if streaming:
- # Special handling for streaming response to accommodate the nature of data flow
awaited_wrapped = wrapped(*args, **kwargs)
span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
- return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
+ return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)
- # Handling for non-streaming responses
else:
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.time()
response = wrapped(*args, **kwargs)
- end_time = time.time()
-
- response_dict = response_as_dict(response)
- try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- input_tokens = response_dict.get('usage').get('input_tokens')
- output_tokens = response_dict.get('usage').get('output_tokens')
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(request_model,
- pricing_info, input_tokens,
- output_tokens)
-
- llm_response = ""
- for i in range(len(response_dict.get('content'))):
- if response_dict.get('content')[i].get('type') == 'text':
- llm_response = response_dict.get('content')[i].get('text')
-
- # Set Span attributes (OTel Semconv)
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- request_model)
- span.set_attribute(SemanticConvetion.SERVER_PORT,
- server_port)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
- kwargs.get("stop_sequences", []))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- kwargs.get("top_k", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response_dict.get('stop_reason')])
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response_dict.get('id'))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
- response_dict.get('model'))
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- input_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- output_tokens)
- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
- server_address)
-
- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- response_dict.get('content')[0].get('type'))
-
- # Set Span attributes (Extra)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- False)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- input_tokens + output_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
- end_time - start_time)
- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
- version)
-
- if capture_message_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,
- },
- )
-
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = create_metrics_attributes(
- service_name=application_name,
- deployment_environment=environment,
- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
- request_model=request_model,
- server_address=server_address,
- server_port=server_port,
- response_model=response_dict.get('model'),
- )
-
- metrics["genai_client_usage_tokens"].record(
- input_tokens + output_tokens, attributes
- )
- metrics["genai_client_operation_duration"].record(
- end_time - start_time, attributes
- )
- metrics["genai_server_ttft"].record(
- end_time - start_time, attributes
- )
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_completion_tokens"].add(output_tokens, attributes)
- metrics["genai_prompt_tokens"].add(input_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- # Return original response
- return response
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- # Return original response
- return response
+ response = process_chat_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ event_provider=event_provider,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ return response
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
index bc960d69..2dc074b5 100644
--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
@@ -4,54 +4,37 @@ Module for monitoring Anthropic API calls.
import logging
import time
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.trace import SpanKind
from openlit.__helpers import (
- get_chat_model_cost,
handle_exception,
- response_as_dict,
- calculate_ttft,
- calculate_tbt,
- create_metrics_attributes,
set_server_address_and_port
)
+from openlit.instrumentation.anthropic.utils import (
+ process_chunk,
+ process_chat_response,
+ process_streaming_chat_response,
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def async_messages(version, environment, application_name, tracer,
+def async_messages(version, environment, application_name, tracer, event_provider,
pricing_info, capture_message_content, metrics, disable_metrics):
"""
- Generates a telemetry wrapper for messages to collect metrics.
-
- Args:
- version: Version of the monitoring package.
- environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Anthropic API.
- tracer: OpenTelemetry tracer for creating spans.
- pricing_info: Information used for calculating the cost of Anthropic usage.
- capture_message_content: Flag indicating whether to trace the actual content.
-
- Returns:
- A function that wraps the chat method to add telemetry.
+ Generates a telemetry wrapper for GenAI function call
"""
- class TracedASyncStream:
+ class TracedAsyncStream:
"""
- Wrapper for streaming responses to collect metrics and trace data.
- Wraps the response to collect message IDs and aggregated response.
-
- This class implements the '__aiter__' and '__anext__' methods that
- handle asynchronous streaming responses.
-
- This class also implements '__aenter__' and '__aexit__' methods that
- handle asynchronous context management protocol.
+ Wrapper for streaming responses to collect telemetry.
"""
+
def __init__(
self,
wrapped,
span,
+ span_name,
kwargs,
server_address,
server_port,
@@ -59,13 +42,18 @@ def async_messages(version, environment, application_name, tracer,
):
self.__wrapped__ = wrapped
self._span = span
- # Placeholder for aggregating streaming response
- self._llmresponse = ""
- self._response_id = ""
- self._response_model = ""
- self._finish_reason = ""
- self._input_tokens = ""
- self._output_tokens = ""
+ self._span_name = span_name
+ self._llmresponse = ''
+ self._response_id = ''
+ self._response_model = ''
+ self._finish_reason = ''
+ self._input_tokens = ''
+ self._output_tokens = ''
+ self._tool_arguments = ''
+ self._tool_id = ''
+ self._tool_name = ''
+ self._tool_calls = None
+ self._response_role = ''
self._args = args
self._kwargs = kwargs
@@ -94,348 +82,68 @@ def async_messages(version, environment, application_name, tracer,
async def __anext__(self):
try:
chunk = await self.__wrapped__.__anext__()
- end_time = time.time()
- # Record the timestamp for the current chunk
- self._timestamps.append(end_time)
-
- if len(self._timestamps) == 1:
- # Calculate time to first chunk
- self._ttft = calculate_ttft(self._timestamps, self._start_time)
-
- chunked = response_as_dict(chunk)
-
- # Collect message IDs and input token from events
- if chunked.get('type') == "message_start":
- self._response_id = chunked.get('message').get('id')
- self._input_tokens = chunked.get('message').get('usage').get('input_tokens')
- self._response_model = chunked.get('message').get('model')
- # Collect message IDs and aggregated response from events
- if chunked.get('type') == "content_block_delta":
- content = chunked.get('delta').get('text')
- if content:
- self._llmresponse += content
- # Collect output tokens and stop reason from events
- if chunked.get('type') == "message_delta":
- self._output_tokens = chunked.get('usage').get('output_tokens')
- self._finish_reason = chunked.get('delta').get('stop_reason')
-
+ process_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- # Handling exception ensure observability without disrupting operation
try:
- self._end_time = time.time()
- if len(self._timestamps) > 1:
- self._tbt = calculate_tbt(self._timestamps)
-
- # Format 'messages' into a single string
- message_prompt = self._kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str_list = []
- for item in content:
- if item["type"] == "text":
- content_str_list.append(f'text: {item["text"]}')
- elif (item["type"] == "image_url" and
- not item["image_url"]["url"].startswith("data:")):
- content_str_list.append(f'image_url: {item["image_url"]["url"]}')
- content_str = ", ".join(content_str_list)
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- request_model = self._kwargs.get("model", "claude-3-5-sonnet-latest")
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(request_model,
- pricing_info, self._input_tokens,
- self._output_tokens)
-
- # Set Span attributes (OTel Semconv)
- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- request_model)
- self._span.set_attribute(SemanticConvetion.SERVER_PORT,
- self._server_port)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- self._kwargs.get("max_tokens", -1))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
- self._kwargs.get("stop_sequences", []))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- self._kwargs.get("temperature", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- self._kwargs.get("top_k", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- self._kwargs.get("top_p", 1.0))
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [self._finish_reason])
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- self._response_id)
- self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
- self._response_model)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- self._input_tokens)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- self._output_tokens)
- self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
- self._server_address)
- if isinstance(self._llmresponse, str):
- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- "text")
- else:
- self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- "json")
-
- # Set Span attributes (Extra)
- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- self._span.set_attribute(SERVICE_NAME,
- application_name)
- self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- True)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- self._input_tokens + self._output_tokens)
- self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,
- self._tbt)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
- self._ttft)
- self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
- version)
- if capture_message_content:
- self._span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- self._span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,
- },
+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ event_provider=event_provider,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version
)
- self._span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = create_metrics_attributes(
- service_name=application_name,
- deployment_environment=environment,
- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
- request_model=request_model,
- server_address=self._server_address,
- server_port=self._server_port,
- response_model=self._response_model,
- )
-
- metrics["genai_client_usage_tokens"].record(
- self._input_tokens + self._output_tokens, attributes
- )
- metrics["genai_client_operation_duration"].record(
- self._end_time - self._start_time, attributes
- )
- metrics["genai_server_tbt"].record(
- self._tbt, attributes
- )
- metrics["genai_server_ttft"].record(
- self._ttft, attributes
- )
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)
- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
except Exception as e:
handle_exception(self._span, e)
logger.error("Error in trace creation: %s", e)
- finally:
- self._span.end()
raise
async def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'messages' API call to add telemetry.
-
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
-
- Args:
- wrapped: The original 'messages' method to be wrapped.
- instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'messages' method.
- kwargs: Keyword arguments for the 'messages' method.
-
- Returns:
- The response from the original 'messages' method.
+ Wraps the GenAI function call.
"""
- # Check if streaming is enabled for the API call
- streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(instance, "api.anthropic.com", 443)
- request_model = kwargs.get("model", "claude-3-5-sonnet-latest")
+ streaming = kwargs.get('stream', False)
+ server_address, server_port = set_server_address_and_port(instance, 'api.anthropic.com', 443)
+ request_model = kwargs.get('model', 'claude-3-5-sonnet-latest')
- span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
+ span_name = f'{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
# pylint: disable=no-else-return
if streaming:
- # Special handling for streaming response to accommodate the nature of data flow
awaited_wrapped = await wrapped(*args, **kwargs)
span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
- return TracedASyncStream(awaited_wrapped, span, kwargs, server_address, server_port)
+ return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)
- # Handling for non-streaming responses
else:
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.time()
response = await wrapped(*args, **kwargs)
- end_time = time.time()
-
- response_dict = response_as_dict(response)
- try:
- # Format 'messages' into a single string
- message_prompt = kwargs.get("messages", "")
- formatted_messages = []
- for message in message_prompt:
- role = message["role"]
- content = message["content"]
-
- if isinstance(content, list):
- content_str = ", ".join(
- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
- if "type" in item else f'text: {item["text"]}'
- for item in content
- )
- formatted_messages.append(f"{role}: {content_str}")
- else:
- formatted_messages.append(f"{role}: {content}")
- prompt = "\n".join(formatted_messages)
-
- input_tokens = response_dict.get('usage').get('input_tokens')
- output_tokens = response_dict.get('usage').get('output_tokens')
-
- # Calculate cost of the operation
- cost = get_chat_model_cost(request_model,
- pricing_info, input_tokens,
- output_tokens)
-
- llm_response = ""
- for i in range(len(response_dict.get('content'))):
- if response_dict.get('content')[i].get('type') == 'text':
- llm_response = response_dict.get('content')[i].get('text')
-
- # Set Span attributes (OTel Semconv)
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,
- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
- SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- request_model)
- span.set_attribute(SemanticConvetion.SERVER_PORT,
- server_port)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
- kwargs.get("max_tokens", -1))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,
- kwargs.get("stop_sequences", []))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- kwargs.get("temperature", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- kwargs.get("top_k", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- kwargs.get("top_p", 1.0))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
- [response_dict.get('stop_reason')])
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
- response_dict.get('id'))
- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,
- response_dict.get('model'))
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,
- input_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,
- output_tokens)
- span.set_attribute(SemanticConvetion.SERVER_ADDRESS,
- server_address)
-
- span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
- response_dict.get('content')[0].get('type'))
-
- # Set Span attributes (Extra)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT,
- environment)
- span.set_attribute(SERVICE_NAME,
- application_name)
- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
- False)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- input_tokens + output_tokens)
- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- cost)
- span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,
- end_time - start_time)
- span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,
- version)
-
- if capture_message_content:
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
- },
- )
- span.add_event(
- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,
- },
- )
-
- span.set_status(Status(StatusCode.OK))
-
- if disable_metrics is False:
- attributes = create_metrics_attributes(
- service_name=application_name,
- deployment_environment=environment,
- operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
- system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
- request_model=request_model,
- server_address=server_address,
- server_port=server_port,
- response_model=response_dict.get('model'),
- )
-
- metrics["genai_client_usage_tokens"].record(
- input_tokens + output_tokens, attributes
- )
- metrics["genai_client_operation_duration"].record(
- end_time - start_time, attributes
- )
- metrics["genai_server_ttft"].record(
- end_time - start_time, attributes
- )
- metrics["genai_requests"].add(1, attributes)
- metrics["genai_completion_tokens"].add(output_tokens, attributes)
- metrics["genai_prompt_tokens"].add(input_tokens, attributes)
- metrics["genai_cost"].record(cost, attributes)
-
- # Return original response
- return response
-
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
-
- # Return original response
- return response
+ response = process_chat_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ event_provider=event_provider,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ return response
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/utils.py b/sdk/python/src/openlit/instrumentation/anthropic/utils.py
new file mode 100644
index 00000000..cfa11c3a
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/anthropic/utils.py
@@ -0,0 +1,251 @@
+"""
+Anthropic OpenTelemetry instrumentation utility functions
+"""
+import time
+
+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.trace import Status, StatusCode
+
+from openlit.__helpers import (
+ calculate_ttft,
+ response_as_dict,
+ calculate_tbt,
+ extract_and_format_input,
+ get_chat_model_cost,
+ create_metrics_attributes,
+ otel_event,
+ concatenate_all_contents
+)
+from openlit.semcov import SemanticConvetion
+
+def process_chunk(self, chunk):
+ """
+ Process a chunk of response data and update state.
+ """
+
+ end_time = time.time()
+ # Record the timestamp for the current chunk
+ self._timestamps.append(end_time)
+
+ if len(self._timestamps) == 1:
+ # Calculate time to first chunk
+ self._ttft = calculate_ttft(self._timestamps, self._start_time)
+
+ chunked = response_as_dict(chunk)
+
+ # Collect message IDs and input token from events
+ if chunked.get('type') == 'message_start':
+ self._response_id = chunked.get('message').get('id')
+ self._input_tokens = chunked.get('message').get('usage').get('input_tokens')
+ self._response_model = chunked.get('message').get('model')
+ self._response_role = chunked.get('message').get('role')
+
+ # Collect message IDs and aggregated response from events
+ if chunked.get('type') == 'content_block_delta':
+ if chunked.get('delta').get('text'):
+ self._llmresponse += chunked.get('delta').get('text')
+ elif chunked.get('delta').get('partial_json'):
+ self._tool_arguments += chunked.get('delta').get('partial_json')
+
+ if chunked.get('type') == 'content_block_start':
+ if chunked.get('content_block').get('id'):
+ self._tool_id = chunked.get('content_block').get('id')
+ if chunked.get('content_block').get('name'):
+ self._tool_name = chunked.get('content_block').get('name')
+
+ # Collect output tokens and stop reason from events
+ if chunked.get('type') == 'message_delta':
+ self._output_tokens = chunked.get('usage').get('output_tokens')
+ self._finish_reason = chunked.get('delta').get('stop_reason')
+
+def common_chat_logic(scope, pricing_info, environment, application_name, metrics,
+ event_provider, capture_message_content, disable_metrics, version, is_stream):
+ """
+ Process chat request and generate Telemetry
+ """
+
+ scope._end_time = time.time()
+ if len(scope._timestamps) > 1:
+ scope._tbt = calculate_tbt(scope._timestamps)
+
+ formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))
+ request_model = scope._kwargs.get('model', 'claude-3-opus-20240229')
+
+ cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)
+
+ # Set Span attributes (OTel Semconv)
+ scope._span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION, SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM, SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL, request_model)
+ scope._span.set_attribute(SemanticConvetion.SERVER_PORT, scope._server_port)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get('max_tokens', -1))
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get('stop_sequences', []))
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get('temperature', 1.0))
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K, scope._kwargs.get('top_k', 1.0))
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P, scope._kwargs.get('top_p', 1.0))
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID, scope._response_id)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL, scope._response_model)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)
+ scope._span.set_attribute(SemanticConvetion.SERVER_ADDRESS, scope._server_address)
+
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,
+ 'text' if isinstance(scope._llmresponse, str) else 'json')
+
+ scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)
+ scope._span.set_attribute(SERVICE_NAME, application_name)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM, is_stream)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST, cost)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT, scope._tbt)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT, scope._ttft)
+ scope._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION, version)
+
+ # To be removed one the change to log events (from span events) is complete
+ prompt = concatenate_all_contents(formatted_messages)
+ if capture_message_content:
+ scope._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,
+ },
+ )
+ scope._span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,
+ },
+ )
+
+ choice_event_body = {
+ 'finish_reason': scope._finish_reason,
+ 'index': 0,
+ 'message': {
+ **({'content': scope._llmresponse} if capture_message_content else {}),
+ 'role': scope._response_role
+ }
+ }
+
+ if scope._tool_calls:
+ choice_event_body['message'].update({
+ 'tool_calls': {
+ 'function': {
+ 'name': scope._tool_calls.get('name', ''),
+ 'arguments': scope._tool_calls.get('input', '')
+ },
+ 'id': scope._tool_calls.get('id', ''),
+ 'type': 'function'
+ }
+ })
+
+ # Emit events
+ for role in ['user', 'system', 'assistant', 'tool']:
+ if formatted_messages.get(role, {}).get('content', ''):
+ event = otel_event(
+ name=getattr(SemanticConvetion, f'GEN_AI_{role.upper()}_MESSAGE'),
+ attributes={
+ SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC
+ },
+ body = {
+ # pylint: disable=line-too-long
+ **({'content': formatted_messages.get(role, {}).get('content', '')} if capture_message_content else {}),
+ 'role': formatted_messages.get(role, {}).get('role', []),
+ **({
+ 'tool_calls': {
+ 'function': {
+ # pylint: disable=line-too-long
+ 'name': (scope._tool_calls[0].get('function', {}).get('name', '') if scope._tool_calls else ''),
+ 'arguments': (scope._tool_calls[0].get('function', {}).get('arguments', '') if scope._tool_calls else '')
+ },
+ 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else ''),
+ 'type': 'function'
+ }
+ } if role == 'assistant' else {}),
+ **({
+ 'id': (scope._tool_calls[0].get('id', '') if scope._tool_calls else '')
+ } if role == 'tool' else {})
+ }
+ )
+ event_provider.emit(event)
+
+ choice_event = otel_event(
+ name=SemanticConvetion.GEN_AI_CHOICE,
+ attributes={
+ SemanticConvetion.GEN_AI_SYSTEM: SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC
+ },
+ body=choice_event_body
+ )
+ event_provider.emit(choice_event)
+
+ scope._span.set_status(Status(StatusCode.OK))
+
+ if not disable_metrics:
+ metrics_attributes = create_metrics_attributes(
+ service_name=application_name,
+ deployment_environment=environment,
+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,
+ system=SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
+ request_model=request_model,
+ server_address=scope._server_address,
+ server_port=scope._server_port,
+ response_model=scope._response_model,
+ )
+
+ metrics['genai_client_usage_tokens'].record(scope._input_tokens + scope._output_tokens, metrics_attributes)
+ metrics['genai_client_operation_duration'].record(scope._end_time - scope._start_time, metrics_attributes)
+ metrics['genai_server_tbt'].record(scope._tbt, metrics_attributes)
+ metrics['genai_server_ttft'].record(scope._ttft, metrics_attributes)
+ metrics['genai_requests'].add(1, metrics_attributes)
+ metrics['genai_completion_tokens'].add(scope._output_tokens, metrics_attributes)
+ metrics['genai_prompt_tokens'].add(scope._input_tokens, metrics_attributes)
+ metrics['genai_cost'].record(cost, metrics_attributes)
+
+def process_streaming_chat_response(self, pricing_info, environment, application_name, metrics,
+ event_provider, capture_message_content=False, disable_metrics=False, version=''):
+ """
+ Process chat request and generate Telemetry
+ """
+ if self._tool_id != '':
+ self._tool_calls = {
+ 'id': self._tool_id,
+ 'name': self._tool_name,
+ 'input': self._tool_arguments
+ }
+
+ common_chat_logic(self, pricing_info, environment, application_name, metrics,
+ event_provider, capture_message_content, disable_metrics, version, is_stream=True)
+
+def process_chat_response(response, request_model, pricing_info, server_port, server_address,
+ environment, application_name, metrics, event_provider, start_time,
+ span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
+ """
+ Process chat request and generate Telemetry
+ """
+
+ self = type('GenericScope', (), {})()
+ response_dict = response_as_dict(response)
+
+ # pylint: disable = no-member
+ self._start_time = start_time
+ self._end_time = time.time()
+ self._span = span
+ self._llmresponse = response_dict.get('content', {})[0].get('text', '')
+ self._response_role = response_dict.get('message', {}).get('role', 'assistant')
+ self._input_tokens = response_dict.get('usage').get('input_tokens')
+ self._output_tokens = response_dict.get('usage').get('output_tokens')
+ self._response_model = response_dict.get('model', '')
+ self._finish_reason = response_dict.get('stop_reason', '')
+ self._response_id = response_dict.get('id', '')
+ self._timestamps = []
+ self._ttft, self._tbt = self._end_time - self._start_time, 0
+ self._server_address, self._server_port = server_address, server_port
+ self._kwargs = kwargs
+ #pylint: disable=line-too-long
+ self._tool_calls = (lambda c: c[1] if len(c) > 1 and c[1].get('type') == 'tool_use' else None)(response_dict.get('content', []))
+
+ common_chat_logic(self, pricing_info, environment, application_name, metrics,
+ event_provider, capture_message_content, disable_metrics, version, is_stream=False)
+
+ return response
diff --git a/sdk/python/src/openlit/instrumentation/ollama/utils.py b/sdk/python/src/openlit/instrumentation/ollama/utils.py
index 4fdb617a..5d259573 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/utils.py
@@ -129,7 +129,6 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
}
if scope._tool_calls:
- print(scope._tool_calls)
function_call = scope._tool_calls[0]
choice_event_body["message"].update({
"tool_calls": {
diff --git a/sdk/python/tests/test_ai21.py b/sdk/python/tests/test_ai21.py
index 70fc1b60..32f76071 100644
--- a/sdk/python/tests/test_ai21.py
+++ b/sdk/python/tests/test_ai21.py
@@ -43,7 +43,7 @@ def test_sync_ai21_chat():
model="jamba-1.5-mini",
max_tokens=1,
)
- assert isinstance(response['id'], str)
+ assert isinstance(response.id, str)
def test_sync_ai21_chat_stream():
"""
@@ -80,7 +80,7 @@ def test_sync_ai21_chat_rag():
max_neighbors=1,
max_tokens=1,
)
- assert isinstance(response['id'], str)
+ assert isinstance(response.id, str)
@pytest.mark.asyncio
async def test_async_ai21_test():
@@ -96,7 +96,7 @@ async def test_async_ai21_test():
model="jamba-1.5-mini",
max_tokens=1,
)
- assert isinstance(response['id'], str)
+ assert isinstance(response.id, str)
responses = await async_client.chat.completions.create(
messages=MESSAGES,
@@ -117,4 +117,4 @@ async def test_async_ai21_test():
max_neighbors=1,
max_tokens=1,
)
- assert isinstance(response['id'], str)
+ assert isinstance(response.id, str)