Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Refactor AI21 OpenTelemetry Monitoring Instrumentation (#765)

Patcher8cf9b9a3167c
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 2c666c06..e180858c 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.34.3"
+version = "1.34.4"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/instrumentation/ai21/__init__.py b/sdk/python/src/openlit/instrumentation/ai21/__init__.py
index 1885fd09..14c67c1b 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/__init__.py
@@ -13,53 +13,52 @@ from openlit.instrumentation.ai21.async_ai21 import (
async_chat, async_chat_rag
)
-_instruments = ('ai21 >= 3.0.0',)
+_instruments = ("ai21 >= 3.0.0",)
class AI21Instrumentor(BaseInstrumentor):
"""
- An instrumentor for AI21's client library.
+ An instrumentor for AI21 client library.
"""
def instrumentation_dependencies(self) -> Collection[str]:
return _instruments
def _instrument(self, **kwargs):
- application_name = kwargs.get('application_name', 'default')
- environment = kwargs.get('environment', 'default')
- tracer = kwargs.get('tracer')
- event_provider = kwargs.get('event_provider')
- metrics = kwargs.get('metrics_dict')
- pricing_info = kwargs.get('pricing_info', {})
- capture_message_content = kwargs.get('capture_message_content', False)
- disable_metrics = kwargs.get('disable_metrics')
- version = importlib.metadata.version('ai21')
+ application_name = kwargs.get("application_name", "default")
+ environment = kwargs.get("environment", "default")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ capture_message_content = kwargs.get("capture_message_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("ai21")
#sync
wrap_function_wrapper(
- 'ai21.clients.studio.resources.chat.chat_completions',
- 'ChatCompletions.create',
+ "ai21.clients.studio.resources.chat.chat_completions",
+ "ChatCompletions.create",
chat(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
wrap_function_wrapper(
- 'ai21.clients.studio.resources.studio_conversational_rag',
- 'StudioConversationalRag.create',
+ "ai21.clients.studio.resources.studio_conversational_rag",
+ "StudioConversationalRag.create",
chat_rag(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
#Async
wrap_function_wrapper(
- 'ai21.clients.studio.resources.chat.async_chat_completions',
- 'AsyncChatCompletions.create',
+ "ai21.clients.studio.resources.chat.async_chat_completions",
+ "AsyncChatCompletions.create",
async_chat(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
wrap_function_wrapper(
- 'ai21.clients.studio.resources.studio_conversational_rag',
- 'AsyncStudioConversationalRag.create',
+ "ai21.clients.studio.resources.studio_conversational_rag",
+ "AsyncStudioConversationalRag.create",
async_chat_rag(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics),
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),
)
def _uninstrument(self, **kwargs):
diff --git a/sdk/python/src/openlit/instrumentation/ai21/ai21.py b/sdk/python/src/openlit/instrumentation/ai21/ai21.py
index 5d8305b1..8a8c6cff 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/ai21.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/ai21.py
@@ -22,7 +22,7 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
def chat(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -46,9 +46,9 @@ def chat(version, environment, application_name,
self._span = span
self._span_name = span_name
# Placeholder for aggregating streaming response
- self._llmresponse = ''
- self._response_id = ''
- self._finish_reason = ''
+ self._llmresponse = ""
+ self._response_id = ""
+ self._finish_reason = ""
self._input_tokens = 0
self._output_tokens = 0
self._choices = []
@@ -92,14 +92,13 @@ def chat(version, environment, application_name,
environment=environment,
application_name=application_name,
metrics=metrics,
- event_provider=event_provider,
capture_message_content=capture_message_content,
disable_metrics=disable_metrics,
version=version
)
except Exception as e:
handle_exception(self._span, e)
- logger.error('Error in trace creation: %s', e)
+ logger.error("Error in trace creation: %s", e)
raise
def wrapper(wrapped, instance, args, kwargs):
@@ -108,12 +107,12 @@ def chat(version, environment, application_name,
"""
# Check if streaming is enabled for the API call
- streaming = kwargs.get('stream', False)
+ streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
- request_model = kwargs.get('model', 'jamba-1.5-mini')
+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
+ request_model = kwargs.get("model", "jamba-1.5-mini")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
# pylint: disable=no-else-return
if streaming:
@@ -127,30 +126,34 @@ def chat(version, environment, application_name,
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = wrapped(*args, **kwargs)
- response = process_chat_response(
- response=response,
- request_model=request_model,
- pricing_info=pricing_info,
- server_port=server_port,
- server_address=server_address,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- start_time=start_time,
- span=span,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- **kwargs
- )
- return response
+ try:
+ response = process_chat_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
return wrapper
def chat_rag(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -160,32 +163,36 @@ def chat_rag(version, environment, application_name,
Wraps the GenAI function call.
"""
- server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
- request_model = kwargs.get('model', 'jamba-1.5-mini')
+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
+ request_model = kwargs.get("model", "jamba-1.5-mini")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = wrapped(*args, **kwargs)
- response = process_chat_rag_response(
- response=response,
- request_model=request_model,
- pricing_info=pricing_info,
- server_port=server_port,
- server_address=server_address,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- start_time=start_time,
- span=span,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- **kwargs
- )
-
- return response
+
+ try:
+ response = process_chat_rag_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
index 3ecd94c1..e820c941 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
@@ -22,7 +22,7 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
def async_chat(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -46,9 +46,9 @@ def async_chat(version, environment, application_name,
self._span = span
self._span_name = span_name
# Placeholder for aggregating streaming response
- self._llmresponse = ''
- self._response_id = ''
- self._finish_reason = ''
+ self._llmresponse = ""
+ self._response_id = ""
+ self._finish_reason = ""
self._input_tokens = 0
self._output_tokens = 0
self._choices = []
@@ -92,14 +92,13 @@ def async_chat(version, environment, application_name,
environment=environment,
application_name=application_name,
metrics=metrics,
- event_provider=event_provider,
capture_message_content=capture_message_content,
disable_metrics=disable_metrics,
version=version
)
except Exception as e:
handle_exception(self._span, e)
- logger.error('Error in trace creation: %s', e)
+
raise
async def wrapper(wrapped, instance, args, kwargs):
@@ -108,12 +107,12 @@ def async_chat(version, environment, application_name,
"""
# Check if streaming is enabled for the API call
- streaming = kwargs.get('stream', False)
+ streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
- request_model = kwargs.get('model', 'jamba-1.5-mini')
+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
+ request_model = kwargs.get("model", "jamba-1.5-mini")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
# pylint: disable=no-else-return
if streaming:
@@ -127,30 +126,34 @@ def async_chat(version, environment, application_name,
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = await wrapped(*args, **kwargs)
- response = process_chat_response(
- response=response,
- request_model=request_model,
- pricing_info=pricing_info,
- server_port=server_port,
- server_address=server_address,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- start_time=start_time,
- span=span,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- **kwargs
- )
- return response
+ try:
+ response = process_chat_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
return wrapper
def async_chat_rag(version, environment, application_name,
- tracer, event_provider, pricing_info, capture_message_content, metrics, disable_metrics):
+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):
"""
Generates a telemetry wrapper for GenAI function call
"""
@@ -160,32 +163,36 @@ def async_chat_rag(version, environment, application_name,
Wraps the GenAI function call.
"""
- server_address, server_port = set_server_address_and_port(instance, 'api.ai21.com', 443)
- request_model = kwargs.get('model', 'jamba-1.5-mini')
+ server_address, server_port = set_server_address_and_port(instance, "api.ai21.com", 443)
+ request_model = kwargs.get("model", "jamba-1.5-mini")
- span_name = f'{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}'
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:
start_time = time.time()
response = await wrapped(*args, **kwargs)
- response = process_chat_rag_response(
- response=response,
- request_model=request_model,
- pricing_info=pricing_info,
- server_port=server_port,
- server_address=server_address,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- event_provider=event_provider,
- start_time=start_time,
- span=span,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- **kwargs
- )
-
- return response
+
+ try:
+ response = process_chat_rag_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/ai21/utils.py b/sdk/python/src/openlit/instrumentation/ai21/utils.py
index d72cfc0c..9c7c2d87 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/utils.py
@@ -14,9 +14,7 @@ from openlit.__helpers import (
general_tokens,
extract_and_format_input,
get_chat_model_cost,
- handle_exception,
create_metrics_attributes,
- otel_event,
concatenate_all_contents
)
from openlit.semcov import SemanticConvention
@@ -29,36 +27,38 @@ def setup_common_span_attributes(span, request_model, kwargs, tokens,
"""
# Base attributes from SDK and operation settings.
- span.set_attribute(TELEMETRY_SDK_NAME, 'openlit')
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)
span.set_attribute(SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_AI21)
span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL, request_model)
span.set_attribute(SemanticConvention.SERVER_PORT, server_port)
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED, kwargs.get('seed', ''))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, kwargs.get('frequency_penalty', 0.0))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, kwargs.get('max_tokens', -1))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, kwargs.get('presence_penalty', 0.0))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, kwargs.get('stop', []))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, kwargs.get('temperature', 0.4))
- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, kwargs.get('top_p', 1.0))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED, kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, kwargs.get("frequency_penalty", 0.0))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, kwargs.get("max_tokens", -1))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, kwargs.get("presence_penalty", 0.0))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, kwargs.get("stop", []))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, kwargs.get("temperature", 0.4))
+ span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, kwargs.get("top_p", 1.0))
# Add token-related attributes if available.
- if 'finish_reason' in tokens:
- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [tokens['finish_reason']])
- if 'response_id' in tokens:
- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, tokens['response_id'])
- if 'input_tokens' in tokens:
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, tokens['input_tokens'])
- if 'output_tokens' in tokens:
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, tokens['output_tokens'])
- if 'total_tokens' in tokens:
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, tokens['total_tokens'])
+ if "finish_reason" in tokens:
+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [tokens["finish_reason"]])
+ if "response_id" in tokens:
+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, tokens["response_id"])
+ if "input_tokens" in tokens:
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, tokens["input_tokens"])
+ if "output_tokens" in tokens:
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, tokens["output_tokens"])
+ if "total_tokens" in tokens:
+ span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, tokens["total_tokens"])
span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, request_model)
span.set_attribute(SemanticConvention.SERVER_ADDRESS, server_address)
+
# Environment and service identifiers.
span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)
span.set_attribute(SERVICE_NAME, application_name)
+
# Set any extra attributes passed in.
for key, value in extra_attrs.items():
span.set_attribute(key, value)
@@ -80,106 +80,15 @@ def record_common_metrics(metrics, application_name, environment, request_model,
server_port=server_port,
response_model=request_model,
)
- metrics['genai_client_usage_tokens'].record(input_tokens + output_tokens, attributes)
- metrics['genai_client_operation_duration'].record(end_time - start_time, attributes)
+ metrics["genai_client_usage_tokens"].record(input_tokens + output_tokens, attributes)
+ metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)
if include_tbt and tbt_value is not None:
- metrics['genai_server_tbt'].record(tbt_value, attributes)
- metrics['genai_server_ttft'].record(end_time - start_time, attributes)
- metrics['genai_requests'].add(1, attributes)
- metrics['genai_completion_tokens'].add(output_tokens, attributes)
- metrics['genai_prompt_tokens'].add(input_tokens, attributes)
- metrics['genai_cost'].record(cost, attributes)
-
-def emit_common_events(event_provider, choices, finish_reason, llmresponse, formatted_messages,
- capture_message_content, n):
- """
- Emit events common to both chat and chat rag operations.
- """
-
- if n > 1:
- for choice in choices:
- choice_event_body = {
- 'finish_reason': finish_reason,
- 'index': choice.get('index', 0),
- 'message': {
- **({'content': choice.get('message', {}).get('content', '')} if capture_message_content else {}),
- 'role': choice.get('message', {}).get('role', 'assistant')
- }
- }
- # If tool calls exist, emit an event for each tool call.
- tool_calls = choice.get('message', {}).get('tool_calls')
- if tool_calls:
- for tool_call in tool_calls:
- choice_event_body['message'].update({
- 'tool_calls': {
- 'function': {
- 'name': tool_call.get('function', {}).get('name', ''),
- 'arguments': tool_call.get('function', {}).get('arguments', '')
- },
- 'id': tool_call.get('id', ''),
- 'type': tool_call.get('type', 'function')
- }
- })
- event = otel_event(
- name=SemanticConvention.GEN_AI_CHOICE,
- attributes={SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AI21},
- body=choice_event_body
- )
- event_provider.emit(event)
- else:
- event = otel_event(
- name=SemanticConvention.GEN_AI_CHOICE,
- attributes={SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AI21},
- body=choice_event_body
- )
- event_provider.emit(event)
- else:
- # Single choice case.
- choice_event_body = {
- 'finish_reason': finish_reason,
- 'index': 0,
- 'message': {
- **({'content': llmresponse} if capture_message_content else {}),
- 'role': 'assistant'
- }
- }
- event = otel_event(
- name=SemanticConvention.GEN_AI_CHOICE,
- attributes={SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AI21},
- body=choice_event_body
- )
- event_provider.emit(event)
-
- # Emit additional role-based events (if formatted messages are available).
- for role in ['user', 'system', 'assistant', 'tool']:
- msg = formatted_messages.get(role, {})
- if msg.get('content', ''):
- event_body = {
- **({'content': msg.get('content', '')} if capture_message_content else {}),
- 'role': msg.get('role', [])
- }
- # For assistant messages, attach tool call details if they exist.
- if role == 'assistant' and choices:
- tool_calls = choices[0].get('message', {}).get('tool_calls', [])
- if tool_calls:
- event_body['tool_calls'] = {
- 'function': {
- 'name': tool_calls[0].get('function', {}).get('name', ''),
- 'arguments': tool_calls[0].get('function', {}).get('arguments', '')
- },
- 'id': tool_calls[0].get('id', ''),
- 'type': 'function'
- }
- if role == 'tool' and choices:
- tool_calls = choices[0].get('message', {}).get('tool_calls', [])
- if tool_calls:
- event_body['id'] = tool_calls[0].get('id', '')
- event = otel_event(
- name=getattr(SemanticConvention, f'GEN_AI_{role.upper()}_MESSAGE'),
- attributes={SemanticConvention.GEN_AI_SYSTEM: SemanticConvention.GEN_AI_SYSTEM_AI21},
- body=event_body
- )
- event_provider.emit(event)
+ metrics["genai_server_tbt"].record(tbt_value, attributes)
+ metrics["genai_server_ttft"].record(end_time - start_time, attributes)
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
def process_chunk(self, chunk):
"""
@@ -194,21 +103,20 @@ def process_chunk(self, chunk):
self._ttft = calculate_ttft(self._timestamps, self._start_time)
chunked = response_as_dict(chunk)
- if (len(chunked.get('choices')) > 0 and
- 'delta' in chunked.get('choices')[0] and
- 'content' in chunked.get('choices')[0].get('delta')):
- content = chunked.get('choices')[0].get('delta').get('content')
- if content:
+ if (len(chunked.get("choices")) > 0 and
+ "delta" in chunked.get("choices")[0] and
+ "content" in chunked.get("choices")[0].get("delta")):
+ if content := chunked.get("choices")[0].get("delta").get("content"):
self._llmresponse += content
- if chunked.get('usage'):
- self._input_tokens = chunked.get('usage').get('prompt_tokens')
- self._output_tokens = chunked.get('usage').get('completion_tokens')
- self._response_id = chunked.get('id')
- self._choices += chunked.get('choices')
- self._finish_reason = chunked.get('choices')[0].get('finish_reason')
+ if chunked.get("usage"):
+ self._input_tokens = chunked.get("usage").get("prompt_tokens")
+ self._output_tokens = chunked.get("usage").get("completion_tokens")
+ self._response_id = chunked.get("id")
+ self._choices += chunked.get("choices")
+ self._finish_reason = chunked.get("choices")[0].get("finish_reason")
def common_chat_logic(scope, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream):
+ capture_message_content, disable_metrics, version, is_stream):
"""
Process chat request and generate Telemetry.
"""
@@ -218,19 +126,19 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
scope._tbt = calculate_tbt(scope._timestamps)
# Extract and format input messages.
- formatted_messages = extract_and_format_input(scope._kwargs.get('messages', ''))
+ formatted_messages = extract_and_format_input(scope._kwargs.get("messages", ""))
prompt = concatenate_all_contents(formatted_messages)
- request_model = scope._kwargs.get('model', 'jamba-1.5-mini')
+ request_model = scope._kwargs.get("model", "jamba-1.5-mini")
# Calculate cost based on token usage.
cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)
# Prepare tokens dictionary.
tokens = {
- 'finish_reason': scope._finish_reason,
- 'response_id': scope._response_id,
- 'input_tokens': scope._input_tokens,
- 'output_tokens': scope._output_tokens,
- 'total_tokens': scope._input_tokens + scope._output_tokens,
+ "finish_reason": scope._finish_reason,
+ "response_id": scope._response_id,
+ "input_tokens": scope._input_tokens,
+ "output_tokens": scope._output_tokens,
+ "total_tokens": scope._input_tokens + scope._output_tokens,
}
extra_attrs = {
SemanticConvention.GEN_AI_REQUEST_IS_STREAM: is_stream,
@@ -239,14 +147,13 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
SemanticConvention.GEN_AI_SERVER_TBT: scope._tbt,
SemanticConvention.GEN_AI_SERVER_TTFT: scope._ttft,
SemanticConvention.GEN_AI_SDK_VERSION: version,
- SemanticConvention.GEN_AI_OUTPUT_TYPE: 'text' if isinstance(scope._llmresponse, str) else 'json'
+ SemanticConvention.GEN_AI_OUTPUT_TYPE: "text" if isinstance(scope._llmresponse, str) else "json"
}
# Set span attributes.
setup_common_span_attributes(scope._span, request_model, scope._kwargs, tokens,
scope._server_port, scope._server_address, environment,
application_name, extra_attrs)
- # Optionally add events capturing the prompt and completion.
if capture_message_content:
scope._span.add_event(
name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
@@ -257,11 +164,6 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
attributes={SemanticConvention.GEN_AI_CONTENT_COMPLETION: scope._llmresponse},
)
- # Emit events for each choice and message role.
- n = scope._kwargs.get('n', 1)
- emit_common_events(event_provider, scope._choices, scope._finish_reason, scope._llmresponse,
- formatted_messages, capture_message_content, n)
-
scope._span.set_status(Status(StatusCode.OK))
if not disable_metrics:
@@ -272,23 +174,23 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric
include_tbt=True, tbt_value=scope._tbt)
def process_streaming_chat_response(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content=False, disable_metrics=False, version=''):
+ capture_message_content=False, disable_metrics=False, version=""):
"""
Process a streaming chat response and generate Telemetry.
"""
common_chat_logic(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream=True)
+ capture_message_content, disable_metrics, version, is_stream=True)
def process_chat_response(response, request_model, pricing_info, server_port, server_address,
- environment, application_name, metrics, event_provider, start_time,
- span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
+ environment, application_name, metrics, start_time,
+ span, capture_message_content=False, disable_metrics=False, version="1.0.0", **kwargs):
"""
Process a synchronous chat response and generate Telemetry.
"""
# Create a generic scope object to hold telemetry data.
- self = type('GenericScope', (), {})()
+ self = type("GenericScope", (), {})()
response_dict = response_as_dict(response)
# pylint: disable = no-member
@@ -297,113 +199,102 @@ def process_chat_response(response, request_model, pricing_info, server_port, se
self._span = span
# Concatenate content from all choices.
- self._llmresponse = ''.join(
- (choice.get('message', {}).get('content') or '')
- for choice in response_dict.get('choices', [])
+ self._llmresponse = "".join(
+ (choice.get("message", {}).get("content") or "")
+ for choice in response_dict.get("choices", [])
)
- self._response_role = response_dict.get('message', {}).get('role', 'assistant')
- self._input_tokens = response_dict.get('usage', {}).get('prompt_tokens', 0)
- self._output_tokens = response_dict.get('usage', {}).get('completion_tokens', 0)
- self._response_id = response_dict.get('id', '')
+ self._response_role = response_dict.get("message", {}).get("role", "assistant")
+ self._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)
+ self._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)
+ self._response_id = response_dict.get("id", "")
self._response_model = request_model
- self._finish_reason = response_dict.get('choices', [{}])[0].get('finish_reason')
+ self._finish_reason = response_dict.get("choices", [{}])[0].get("finish_reason")
self._timestamps = []
self._ttft, self._tbt = self._end_time - self._start_time, 0
self._server_address, self._server_port = server_address, server_port
self._kwargs = kwargs
- self._choices = response_dict.get('choices')
+ self._choices = response_dict.get("choices")
common_chat_logic(self, pricing_info, environment, application_name, metrics,
- event_provider, capture_message_content, disable_metrics, version, is_stream=False)
+ capture_message_content, disable_metrics, version, is_stream=False)
return response
def process_chat_rag_response(response, request_model, pricing_info, server_port, server_address,
- environment, application_name, metrics, event_provider, start_time,
- span, capture_message_content=False, disable_metrics=False, version='1.0.0', **kwargs):
+ environment, application_name, metrics, start_time,
+ span, capture_message_content=False, disable_metrics=False, version="1.0.0", **kwargs):
"""
Process a chat response and generate Telemetry.
"""
end_time = time.time()
response_dict = response_as_dict(response)
- try:
- # Format input messages into a single prompt string.
- messages_input = kwargs.get('messages', '')
- formatted_messages = extract_and_format_input(messages_input)
- prompt = concatenate_all_contents(formatted_messages)
- input_tokens = general_tokens(prompt)
-
- # Create tokens dict and RAG-specific extra attributes.
- tokens = {'response_id': response_dict.get('id'), 'input_tokens': input_tokens}
- extra_attrs = {
- SemanticConvention.GEN_AI_REQUEST_IS_STREAM: False,
- SemanticConvention.GEN_AI_SERVER_TTFT: end_time - start_time,
- SemanticConvention.GEN_AI_SDK_VERSION: version,
- SemanticConvention.GEN_AI_RAG_MAX_SEGMENTS: kwargs.get('max_segments', -1),
- SemanticConvention.GEN_AI_RAG_STRATEGY: kwargs.get('retrieval_strategy', 'segments'),
- SemanticConvention.GEN_AI_RAG_SIMILARITY_THRESHOLD: kwargs.get('retrieval_similarity_threshold', -1),
- SemanticConvention.GEN_AI_RAG_MAX_NEIGHBORS: kwargs.get('max_neighbors', -1),
- SemanticConvention.GEN_AI_RAG_FILE_IDS: str(kwargs.get('file_ids', '')),
- SemanticConvention.GEN_AI_RAG_DOCUMENTS_PATH: kwargs.get('path', '')
- }
- # Set common span attributes.
- setup_common_span_attributes(span, request_model, kwargs, tokens,
- server_port, server_address, environment, application_name,
- extra_attrs)
-
- # Record the prompt event if requested.
- if capture_message_content:
- span.add_event(
- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
- attributes={SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt},
- )
-
- output_tokens = 0
- choices = response_dict.get('choices', [])
- # Instead of adding a separate event per choice, we aggregate all completion content.
- aggregated_completion = []
- for i in range(kwargs.get('n', 1)):
- # Get the response content from each choice and count tokens.
- content = choices[i].get('content', '')
- aggregated_completion.append(content)
- output_tokens += general_tokens(content)
- if kwargs.get('tools'):
- span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALLS,
- str(choices[i].get('message', {}).get('tool_calls')))
- # Set output type based on actual content type.
- if isinstance(content, str):
- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, 'text')
- elif content is not None:
- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, 'json')
-
- # Concatenate completion responses.
- llmresponse = ''.join(aggregated_completion)
- tokens['output_tokens'] = output_tokens
- tokens['total_tokens'] = input_tokens + output_tokens
-
- cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)
- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, input_tokens + output_tokens)
-
- span.set_status(Status(StatusCode.OK))
- # Emit a single aggregated completion event.
- if capture_message_content:
- span.add_event(
- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,
- attributes={SemanticConvention.GEN_AI_CONTENT_COMPLETION: llmresponse},
- )
- # Emit the rest of the events (choice and role-based events) as before.
- n = kwargs.get('n', 1)
- emit_common_events(event_provider, choices, choices[0].get('finish_reason', ''),
- llmresponse, formatted_messages, capture_message_content, n)
-
- if not disable_metrics:
- record_common_metrics(metrics, application_name, environment, request_model,
- server_address, server_port, start_time, end_time,
- input_tokens, output_tokens, cost, include_tbt=False)
- return response
-
- except Exception as e:
- handle_exception(span, e)
- return response
+ # Format input messages into a single prompt string.
+ messages_input = kwargs.get("messages", "")
+ formatted_messages = extract_and_format_input(messages_input)
+ prompt = concatenate_all_contents(formatted_messages)
+ input_tokens = general_tokens(prompt)
+
+ # Create tokens dict and RAG-specific extra attributes.
+ tokens = {"response_id": response_dict.get("id"), "input_tokens": input_tokens}
+ extra_attrs = {
+ SemanticConvention.GEN_AI_REQUEST_IS_STREAM: False,
+ SemanticConvention.GEN_AI_SERVER_TTFT: end_time - start_time,
+ SemanticConvention.GEN_AI_SDK_VERSION: version,
+ SemanticConvention.GEN_AI_RAG_MAX_SEGMENTS: kwargs.get("max_segments", -1),
+ SemanticConvention.GEN_AI_RAG_STRATEGY: kwargs.get("retrieval_strategy", "segments"),
+ SemanticConvention.GEN_AI_RAG_SIMILARITY_THRESHOLD: kwargs.get("retrieval_similarity_threshold", -1),
+ SemanticConvention.GEN_AI_RAG_MAX_NEIGHBORS: kwargs.get("max_neighbors", -1),
+ SemanticConvention.GEN_AI_RAG_FILE_IDS: str(kwargs.get("file_ids", "")),
+ SemanticConvention.GEN_AI_RAG_DOCUMENTS_PATH: kwargs.get("path", "")
+ }
+ # Set common span attributes.
+ setup_common_span_attributes(span, request_model, kwargs, tokens,
+ server_port, server_address, environment, application_name,
+ extra_attrs)
+
+ if capture_message_content:
+ span.add_event(
+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt},
+ )
+
+ output_tokens = 0
+ choices = response_dict.get("choices", [])
+ aggregated_completion = []
+ for i in range(kwargs.get("n", 1)):
+ # Get the response content from each choice and count tokens.
+ content = choices[i].get("content", "")
+ aggregated_completion.append(content)
+ output_tokens += general_tokens(content)
+ if kwargs.get("tools"):
+ span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALLS,
+ str(choices[i].get("message", {}).get("tool_calls")))
+ # Set output type based on actual content type.
+ if isinstance(content, str):
+ span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text")
+ elif content is not None:
+ span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "json")
+
+ # Concatenate completion responses.
+ llmresponse = "".join(aggregated_completion)
+ tokens["output_tokens"] = output_tokens
+ tokens["total_tokens"] = input_tokens + output_tokens
+
+ cost = get_chat_model_cost(request_model, pricing_info, input_tokens, output_tokens)
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)
+ span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, input_tokens + output_tokens)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if capture_message_content:
+ span.add_event(
+ name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={SemanticConvention.GEN_AI_CONTENT_COMPLETION: llmresponse},
+ )
+
+ if not disable_metrics:
+ record_common_metrics(metrics, application_name, environment, request_model,
+ server_address, server_port, start_time, end_time,
+ input_tokens, output_tokens, cost, include_tbt=False)
+ return response