Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Refactor Groq OpenTelemetry Monitoring Instrumentation (#774)
Patcher1c7987f6ee0a
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 7d76844a..4e0ec050 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.34.10"+version = "1.34.11"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex bb47eb5a..3b592fe1 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -379,3 +379,26 @@ def record_completion_metrics(metrics, gen_ai_operation, gen_ai_system, server_ametrics["genai_completion_tokens"].add(output_tokens, attributes)metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)++def record_embedding_metrics(metrics, gen_ai_operation, gen_ai_system, server_address, server_port,+ request_model, response_model, environment, application_name, start_time, end_time,+ input_tokens, cost):+ """+ Record embedding-specific metrics for the operation.+ """++ attributes = create_metrics_attributes(+ operation=gen_ai_operation,+ system=gen_ai_system,+ server_address=server_address,+ server_port=server_port,+ request_model=request_model,+ response_model=response_model,+ service_name=application_name,+ deployment_environment=environment,+ )+ metrics["genai_client_usage_tokens"].record(input_tokens, attributes)+ metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)diff --git a/sdk/python/src/openlit/instrumentation/groq/__init__.py b/sdk/python/src/openlit/instrumentation/groq/__init__.pyindex 1605bfed..25578aed 100644--- a/sdk/python/src/openlit/instrumentation/groq/__init__.py+++ b/sdk/python/src/openlit/instrumentation/groq/__init__.py@@ -1,4 +1,3 @@-# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code"""Initializer of Auto Instrumentation of Groq Functions"""from typing import Collection@@ -13,15 +12,15 @@ _instruments = ("groq >= 0.5.0",)class GroqInstrumentor(BaseInstrumentor):"""- An instrumentor for Groq's client library.+ An instrumentor for Groq client library."""def instrumentation_dependencies(self) -> Collection[str]:return _instrumentsdef _instrument(self, **kwargs):- application_name = kwargs.get("application_name", "default_application")- environment = kwargs.get("environment", "default_environment")+ application_name = kwargs.get("application_name", "default")+ environment = kwargs.get("environment", "default")tracer = kwargs.get("tracer")metrics = kwargs.get("metrics_dict")pricing_info = kwargs.get("pricing_info", {})@@ -29,22 +28,21 @@ class GroqInstrumentor(BaseInstrumentor):disable_metrics = kwargs.get("disable_metrics")version = importlib.metadata.version("groq")- #sync+ # Chat completionswrap_function_wrapper("groq.resources.chat.completions","Completions.create",chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)- #async+ # Chat completionswrap_function_wrapper("groq.resources.chat.completions","AsyncCompletions.create",async_chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)def _uninstrument(self, **kwargs):- # Proper uninstrumentation logic to revert patched methodspassdiff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.pyindex 397dcebd..b66b5fb6 100644--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py@@ -1,57 +1,36 @@"""-Module for monitoring Groq API calls.+Module for monitoring Groq API calls (async version)."""-import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,set_server_address_and_port)+from openlit.instrumentation.groq.utils import (+ process_chunk,+ process_streaming_chat_response,+ process_chat_response+)from openlit.semcov import SemanticConvention-# Initialize logger for logging potential issues and operations-logger = logging.getLogger(__name__)--def async_chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+def async_chat(version, environment, application_name, tracer, pricing_info,+ capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat completions to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Groq API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Groq usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat completions method to add telemetry.+ Generates a telemetry wrapper for GenAI function call"""class TracedAsyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for async streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -59,15 +38,15 @@ def async_chat(version, environment, application_name,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""self._response_id = ""self._response_model = ""self._finish_reason = ""+ self._tools = Noneself._system_fingerprint = ""self._input_tokens = 0self._output_tokens = 0-self._args = argsself._kwargs = kwargsself._start_time = time.time()@@ -95,373 +74,70 @@ def async_chat(version, environment, application_name,async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from events- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and- 'content' in chunked.get('choices')[0].get('delta'))):-- content = chunked.get('choices')[0].get('delta').get('content')- if content:- self._llmresponse += content-- if chunked.get('usage'):- self._input_tokens = chunked.get('usage').get('prompt_tokens')- self._output_tokens = chunked.get('usage').get('completion_tokens')- self._response_id = chunked.get('id')- self._response_model = chunked.get('model')- self._finish_reason = chunked.get('choices')[0].get('finish_reason')- self._system_fingerprint = chunked.get('system_fingerprint')+ process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "gpt-4o")-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_GROQ)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_completion_tokens", -1))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop", []))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- self._response_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SERVICE_TIER,- self._kwargs.get("service_tier", "on_demand"))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- self._system_fingerprint)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER,- self._kwargs.get("user", ""))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_GROQ,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._response_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)except Exception as e:handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()+raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat.completions' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.-- Returns:- The response from the original 'chat.completions' method.+ Wraps the GenAI function call."""-# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "api.groq.com", 443)- request_model = kwargs.get("model", "gpt-4o")+ request_model = kwargs.get("model", "mixtral-8x7b-32768")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flow+ # Special handling for streaming responseawaited_wrapped = await wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)-- return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)-- # Handling for non-streaming responses+ return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)else:- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ # Handling for non-streaming responses+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = await wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('prompt_tokens')- output_tokens = response_dict.get('usage').get('completion_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_GROQ)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_completion_tokens", -1))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop", []))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- response_dict.get('model'))- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SERVICE_TIER,- kwargs.get("service_tier", "on_demand"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- response_dict.get('system_fingerprint'))-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- for i in range(kwargs.get('n',1)):- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices')[i].get('finish_reason')])- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvention.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),- },- )- if kwargs.get('tools'):- span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALLS,- str(response_dict.get('choices')[i].get('message').get('tool_calls')))-- if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- elif response_dict.get('choices')[i].get('message').get('content') is not None:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_GROQ,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=response_dict.get('model'),- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )except Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)- # Return original response- return response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/groq/groq.py b/sdk/python/src/openlit/instrumentation/groq/groq.pyindex 85d23483..15da7708 100644--- a/sdk/python/src/openlit/instrumentation/groq/groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/groq.py@@ -2,56 +2,35 @@Module for monitoring Groq API calls."""-import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,set_server_address_and_port)+from openlit.instrumentation.groq.utils import (+ process_chunk,+ process_streaming_chat_response,+ process_chat_response+)from openlit.semcov import SemanticConvention-# Initialize logger for logging potential issues and operations-logger = logging.getLogger(__name__)--def chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+def chat(version, environment, application_name, tracer, pricing_info,+ capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat completions to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Groq API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Groq usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat completions method to add telemetry.+ Generates a telemetry wrapper for GenAI function call"""class TracedSyncStream:"""- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.+ Wrapper for streaming responses to collect telemetry."""+def __init__(self,wrapped,span,+ span_name,kwargs,server_address,server_port,@@ -59,15 +38,15 @@ def chat(version, environment, application_name,):self.__wrapped__ = wrappedself._span = span- # Placeholder for aggregating streaming response+ self._span_name = span_nameself._llmresponse = ""self._response_id = ""self._response_model = ""self._finish_reason = ""+ self._tools = Noneself._system_fingerprint = ""self._input_tokens = 0self._output_tokens = 0-self._args = argsself._kwargs = kwargsself._start_time = time.time()@@ -95,373 +74,70 @@ def chat(version, environment, application_name,def __next__(self):try:chunk = self.__wrapped__.__next__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)- # Collect message IDs and aggregated response from events- if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and- 'content' in chunked.get('choices')[0].get('delta'))):-- content = chunked.get('choices')[0].get('delta').get('content')- if content:- self._llmresponse += content-- if chunked.get('usage'):- self._input_tokens = chunked.get('usage').get('prompt_tokens')- self._output_tokens = chunked.get('usage').get('completion_tokens')- self._response_id = chunked.get('id')- self._response_model = chunked.get('model')- self._finish_reason = chunked.get('choices')[0].get('finish_reason')- self._system_fingerprint = chunked.get('system_fingerprint')+ process_chunk(self, chunk)return chunkexcept StopIteration:- # Handling exception ensure observability without disrupting operationtry:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "gpt-4o")-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_GROQ)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_completion_tokens", -1))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop", []))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("top_p", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- self._response_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SERVICE_TIER,- self._kwargs.get("service_tier", "on_demand"))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- self._system_fingerprint)- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER,- self._kwargs.get("user", ""))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_GROQ,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=self._response_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version)- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)except Exception as e:handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()+raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat.completions' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.-- Returns:- The response from the original 'chat.completions' method.+ Wraps the GenAI function call."""-# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "api.groq.com", 443)- request_model = kwargs.get("model", "gpt-4o")+ request_model = kwargs.get("model", "mixtral-8x7b-32768")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- # pylint: disable=no-else-returnif streaming:- # Special handling for streaming response to accommodate the nature of data flow+ # Special handling for streaming responseawaited_wrapped = wrapped(*args, **kwargs)span = tracer.start_span(span_name, kind=SpanKind.CLIENT)-- return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)-- # Handling for non-streaming responses+ return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)else:- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ # Handling for non-streaming responses+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('prompt_tokens')- output_tokens = response_dict.get('usage').get('completion_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, input_tokens,- output_tokens)-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_GROQ)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_completion_tokens", -1))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop", []))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- response_dict.get('model'))- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SERVICE_TIER,- kwargs.get("service_tier", "on_demand"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,- response_dict.get('system_fingerprint'))-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )-- for i in range(kwargs.get('n',1)):- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [response_dict.get('choices')[i].get('finish_reason')])- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- # pylint: disable=line-too-long- SemanticConvention.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),- },- )- if kwargs.get('tools'):- span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALLS,- str(response_dict.get('choices')[i].get('message').get('tool_calls')))-- if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- elif response_dict.get('choices')[i].get('message').get('content') is not None:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_GROQ,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=response_dict.get('model'),- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )except Exception as e:handle_exception(span, e)- logger.error("Error in trace creation: %s", e)- # Return original response- return response+ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/groq/utils.py b/sdk/python/src/openlit/instrumentation/groq/utils.pynew file mode 100644index 00000000..23c9980b--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/groq/utils.py@@ -0,0 +1,199 @@+"""+Groq OpenTelemetry instrumentation utility functions+"""+import time++from opentelemetry.trace import Status, StatusCode++from openlit.__helpers import (+ calculate_ttft,+ response_as_dict,+ calculate_tbt,+ get_chat_model_cost,+ common_span_attributes,+ record_completion_metrics,+)+from openlit.semcov import SemanticConvention++def format_content(messages):+ """+ Process a list of messages to extract content.+ """++ formatted_messages = []+ for message in messages:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")++ return "\n".join(formatted_messages)++def process_chunk(scope, chunk):+ """+ Process a chunk of response data and update state.+ """++ end_time = time.time()+ # Record the timestamp for the current chunk+ scope._timestamps.append(end_time)++ if len(scope._timestamps) == 1:+ # Calculate time to first chunk+ scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)++ chunked = response_as_dict(chunk)++ # Collect message IDs and aggregated response from events+ if (len(chunked.get("choices", [])) > 0 and+ "delta" in chunked.get("choices")[0] and+ "content" in chunked.get("choices")[0].get("delta", {})):++ content = chunked.get("choices")[0].get("delta").get("content")+ if content:+ scope._llmresponse += content++ if chunked.get('x_groq') is not None:+ if chunked.get('x_groq').get('usage') is not None:+ scope._input_tokens = chunked.get('x_groq').get('usage').get('prompt_tokens')+ scope._output_tokens = chunked.get('x_groq').get('usage').get('completion_tokens')+ scope._response_id = chunked.get('x_groq').get('id')+ scope._response_model = chunked.get('x_groq').get('model')+ scope._finish_reason = chunked.get('choices', [{}])[0].get('finish_reason')+ scope._system_fingerprint = chunked.get('x_groq').get('system_fingerprint')+ scope._end_time = time.time()+++def common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream):+ """+ Process chat request and generate Telemetry+ """++ if len(scope._timestamps) > 1:+ scope._tbt = calculate_tbt(scope._timestamps)++ prompt = format_content(scope._kwargs.get("messages", []))+ request_model = scope._kwargs.get("model", "mixtral-8x7b-32768")++ cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)++ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_GROQ,+ scope._server_address, scope._server_port, request_model, scope._response_model,+ environment, application_name, is_stream, scope._tbt, scope._ttft, version)++ # Span Attributes for Request parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED, scope._kwargs.get("seed", ""))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, scope._kwargs.get("frequency_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get("max_completion_tokens", -1))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, scope._kwargs.get("presence_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get("stop", []))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get("temperature", 1.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get("top_p", 1.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_USER, scope._kwargs.get("user", ""))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SERVICE_TIER, scope._kwargs.get("service_tier", "on_demand"))++ # Span Attributes for Response parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, scope._response_id)+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT, scope._system_fingerprint)+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text" if isinstance(scope._llmresponse, str) else "json")++ # Span Attributes for Cost and Tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)++ # Span Attributes for Tools+ if scope._tools:+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, scope._tools.get("function", {}).get("name", ""))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, str(scope._tools.get("id", "")))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, str(scope._tools.get("function", {}).get("arguments", "")))++ # Span Attributes for Content+ if capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse)++ # To be removed one the change to span_attributes (from span events) is complete+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,+ },+ )++ scope._span.set_status(Status(StatusCode.OK))++ # Metrics+ if not disable_metrics:+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_GROQ,+ scope._server_address, scope._server_port, request_model, scope._response_model, environment,+ application_name, scope._start_time, scope._end_time, scope._input_tokens, scope._output_tokens,+ cost, scope._tbt, scope._ttft)++def process_streaming_chat_response(scope, pricing_info, environment, application_name, metrics,+ capture_message_content=False, disable_metrics=False, version=""):+ """+ Process chat request and generate Telemetry+ """++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=True)++def process_chat_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, start_time, span, capture_message_content=False,+ disable_metrics=False, version="1.0.0", **kwargs):+ """+ Process chat request and generate Telemetry+ """++ # Create scope object+ scope = type("GenericScope", (), {})()+ response_dict = response_as_dict(response)++ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ scope._llmresponse = " ".join(+ (choice.get("message", {}).get("content") or "")+ for choice in response_dict.get("choices", [])+ )+ scope._response_id = response_dict.get("id")+ scope._response_model = response_dict.get("model")+ scope._input_tokens = response_dict.get("usage", {}).get("prompt_tokens", 0)+ scope._output_tokens = response_dict.get("usage", {}).get("completion_tokens", 0)+ scope._timestamps = []+ scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0+ scope._server_address, scope._server_port = server_address, server_port+ scope._kwargs = kwargs+ scope._system_fingerprint = response_dict.get("system_fingerprint")+ scope._finish_reason = str(response_dict.get("choices", [])[0].get("finish_reason", ""))++ # Handle tool calls+ if scope._kwargs.get("tools"):+ scope._tools = response_dict.get("choices", [{}])[0].get("message", {}).get("tool_calls")+ else:+ scope._tools = None++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=False)++ return responsediff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.pyindex 27b6fa46..5ce7de9b 100644--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py@@ -42,7 +42,7 @@ def async_chat(version, environment, application_name,self._llmresponse = ""self._response_model = ""self._finish_reason = ""- self._tool_calls = []+ self._tools = []self._input_tokens = 0self._output_tokens = 0self._response_role = ""@@ -102,7 +102,8 @@ def async_chat(version, environment, application_name,streaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)- request_model = kwargs.get("model")+ json_body = kwargs.get("json", {}) or {}+ request_model = json_body.get("model") or kwargs.get("model")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.pyindex 1e7430a2..0b1a9ebe 100644--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py@@ -42,7 +42,7 @@ def chat(version, environment, application_name,self._llmresponse = ""self._response_model = ""self._finish_reason = ""- self._tool_calls = []+ self._tools = []self._input_tokens = 0self._output_tokens = 0self._response_role = ""@@ -102,7 +102,8 @@ def chat(version, environment, application_name,streaming = kwargs.get("stream", False)server_address, server_port = set_server_address_and_port(instance, "127.0.0.1", 11434)- request_model = kwargs.get("model")+ json_body = kwargs.get("json", {}) or {}+ request_model = json_body.get("model") or kwargs.get("model")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"diff --git a/sdk/python/src/openlit/instrumentation/ollama/utils.py b/sdk/python/src/openlit/instrumentation/ollama/utils.pyindex 9f877ddb..7bf2dd16 100644--- a/sdk/python/src/openlit/instrumentation/ollama/utils.py+++ b/sdk/python/src/openlit/instrumentation/ollama/utils.py@@ -57,7 +57,7 @@ def process_chunk(self, chunk):self._llmresponse += chunked.get("message", {}).get("content", "")if chunked.get("message", {}).get("tool_calls"):- self._tool_calls = chunked["message"]["tool_calls"]+ self._tools = chunked["message"]["tool_calls"]if chunked.get("eval_count"):self._response_role = chunked.get("message", {}).get("role", "")@@ -138,10 +138,10 @@ def common_chat_logic(scope, gen_ai_endpoint, pricing_info, environment, applicascope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)# Span Attributes for Tools- if hasattr(scope, "_tool_calls") and scope._tool_calls:- tool_call = scope._tool_calls[0]- scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, tool_call.get("function", {}).get("name", ""))- scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, str(tool_call.get("function", {}).get("arguments", "")))+ if scope._tools is not None:+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, scope._tools.get("function","")).get("name","")+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, str(scope._tools.get("id","")))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, str(scope._tools.get("function","").get("arguments","")))# Span Attributes for Contentif capture_message_content:@@ -244,7 +244,11 @@ def process_chat_response(response, gen_ai_endpoint, pricing_info, server_port,scope._tbt = 0scope._server_address, scope._server_port = server_address, server_portscope._kwargs = kwargs- scope._tool_calls = response_dict.get("message", {}).get("tool_calls", [])++ if scope._kwargs.get("tools"):+ scope._tools = response_dict.get("choices")[0].get("message").get("tool_calls")+ else:+ scope._tools = Nonecommon_chat_logic(scope, gen_ai_endpoint, pricing_info, environment, application_name, metrics,capture_message_content, disable_metrics, version)diff --git a/sdk/python/src/openlit/instrumentation/premai/utils.py b/sdk/python/src/openlit/instrumentation/premai/utils.pyindex 4a88ca30..17e7b7ac 100644--- a/sdk/python/src/openlit/instrumentation/premai/utils.py+++ b/sdk/python/src/openlit/instrumentation/premai/utils.py@@ -3,7 +3,6 @@ PremAI OpenTelemetry instrumentation utility functions"""import time-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom opentelemetry.trace import Status, StatusCodefrom openlit.__helpers import (@@ -13,7 +12,9 @@ from openlit.__helpers import (get_chat_model_cost,get_embed_model_cost,general_tokens,- create_metrics_attributes,+ common_span_attributes,+ record_completion_metrics,+ record_embedding_metrics,)from openlit.semcov import SemanticConvention@@ -66,77 +67,6 @@ def process_chunk(scope, chunk):scope._response_model = chunked.get("model")scope._end_time = time.time()-def common_span_attributes(scope, gen_ai_operation, gen_ai_system, server_address, server_port,- request_model, response_model, environment, application_name, is_stream, tbt, ttft, version):- """- Set common span attributes for both chat and RAG operations.- """-- scope._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- scope._span.set_attribute(SemanticConvention.GEN_AI_OPERATION, gen_ai_operation)- scope._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM, gen_ai_system)- scope._span.set_attribute(SemanticConvention.SERVER_ADDRESS, server_address)- scope._span.set_attribute(SemanticConvention.SERVER_PORT, server_port)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL, request_model)- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, response_model)- scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)- scope._span.set_attribute(SERVICE_NAME, application_name)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, is_stream)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT, tbt)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft)- scope._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, version)--def record_completion_metrics(metrics, gen_ai_operation, gen_ai_system, server_address, server_port,- request_model, response_model, environment, application_name, start_time, end_time,- input_tokens, output_tokens, cost, tbt=None, ttft=None):- """- Record completion-specific metrics for the operation.- """-- attributes = create_metrics_attributes(- operation=gen_ai_operation,- system=gen_ai_system,- server_address=server_address,- server_port=server_port,- request_model=request_model,- response_model=response_model,- service_name=application_name,- deployment_environment=environment,- )- metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_client_usage_tokens"].record(input_tokens + output_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)- if tbt is not None:- metrics["genai_server_tbt"].record(tbt, attributes)- if ttft is not None:- metrics["genai_server_ttft"].record(ttft, attributes)--def record_embedding_metrics(metrics, gen_ai_operation, gen_ai_system, server_address, server_port,- request_model, response_model, environment, application_name, start_time, end_time,- input_tokens, cost):- """- Record embedding-specific metrics for the operation.- """-- attributes = create_metrics_attributes(- operation=gen_ai_operation,- system=gen_ai_system,- server_address=server_address,- server_port=server_port,- request_model=request_model,- response_model=response_model,- service_name=application_name,- deployment_environment=environment,- )- metrics["genai_client_usage_tokens"].record(input_tokens, attributes)- metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-def common_chat_logic(scope, pricing_info, environment, application_name, metrics,capture_message_content, disable_metrics, version, is_stream):"""diff --git a/sdk/python/src/openlit/instrumentation/reka/utils.py b/sdk/python/src/openlit/instrumentation/reka/utils.pyindex 0f46c7db..e6a69986 100644--- a/sdk/python/src/openlit/instrumentation/reka/utils.py+++ b/sdk/python/src/openlit/instrumentation/reka/utils.py@@ -3,14 +3,14 @@ Reka OpenTelemetry instrumentation utility functions"""import time-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENTfrom opentelemetry.trace import Status, StatusCodefrom openlit.__helpers import (response_as_dict,calculate_tbt,get_chat_model_cost,- create_metrics_attributes,+ common_span_attributes,+ record_completion_metrics,)from openlit.semcov import SemanticConvention@@ -36,54 +36,6 @@ def format_content(messages):return "\n".join(formatted_messages)-def common_span_attributes(scope, gen_ai_operation, gen_ai_system, server_address, server_port,- request_model, response_model, environment, application_name, is_stream, tbt, ttft, version):- """- Set common span attributes for both chat and RAG operations.- """-- scope._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- scope._span.set_attribute(SemanticConvention.GEN_AI_OPERATION, gen_ai_operation)- scope._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM, gen_ai_system)- scope._span.set_attribute(SemanticConvention.SERVER_ADDRESS, server_address)- scope._span.set_attribute(SemanticConvention.SERVER_PORT, server_port)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL, request_model)- scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, response_model)- scope._span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)- scope._span.set_attribute(SERVICE_NAME, application_name)- scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, is_stream)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT, tbt)- scope._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft)- scope._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, version)--def record_common_metrics(metrics, gen_ai_operation, gen_ai_system, server_address, server_port,- request_model, response_model, environment, application_name, start_time, end_time,- input_tokens, output_tokens, cost, tbt=None, ttft=None):- """- Record common metrics for the operation.- """-- attributes = create_metrics_attributes(- operation=gen_ai_operation,- system=gen_ai_system,- server_address=server_address,- server_port=server_port,- request_model=request_model,- response_model=response_model,- service_name=application_name,- deployment_environment=environment,- )- metrics["genai_client_operation_duration"].record(end_time - start_time, attributes)- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_client_usage_tokens"].record(input_tokens + output_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)- if tbt is not None:- metrics["genai_server_tbt"].record(tbt, attributes)- if ttft is not None:- metrics["genai_server_ttft"].record(ttft, attributes)-def common_chat_logic(scope, pricing_info, environment, application_name, metrics,capture_message_content, disable_metrics, version, is_stream):"""@@ -151,7 +103,7 @@ def common_chat_logic(scope, pricing_info, environment, application_name, metric# Metricsif not disable_metrics:- record_common_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_REKAAI,+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_REKAAI,scope._server_address, scope._server_port, request_model, scope._response_model, environment,application_name, scope._start_time, scope._end_time, scope._input_tokens, scope._output_tokens,cost, scope._tbt, scope._ttft)