Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update GPT4All Instrumentation to align with OTel Semconv (#622)
Patcher2589d0d1051a
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex dde0383f..eea7ee41 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.9"+version = "1.33.10"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.pyindex b19578da..dbe787e3 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/async_azure_ai_inference.py@@ -191,7 +191,7 @@ def async_complete(version, environment, application_name,output_tokens)self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,self._system_fingerprint)if isinstance(self._llmresponse, str):self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,@@ -370,7 +370,7 @@ def async_complete(version, environment, application_name,output_tokens)span.set_attribute(SemanticConvetion.SERVER_ADDRESS,server_address)- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,response_dict.get('system_fingerprint'))# Set base span attribues (Extras)diff --git a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.pyindex f5c56d1f..6e81c746 100644--- a/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py+++ b/sdk/python/src/openlit/instrumentation/azure_ai_inference/azure_ai_inference.py@@ -191,7 +191,7 @@ def complete(version, environment, application_name,output_tokens)self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,self._system_fingerprint)if isinstance(self._llmresponse, str):self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,@@ -370,7 +370,7 @@ def complete(version, environment, application_name,output_tokens)span.set_attribute(SemanticConvetion.SERVER_ADDRESS,server_address)- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,response_dict.get('system_fingerprint'))# Set base span attribues (Extras)diff --git a/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.pyindex 54625787..4b4ba150 100644--- a/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py+++ b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py@@ -34,7 +34,7 @@ class GPT4AllInstrumentor(BaseInstrumentor):wrap_function_wrapper("gpt4all","GPT4All.generate",- generate("gpt4all.generate", version, environment, application_name,+ generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -42,7 +42,7 @@ class GPT4AllInstrumentor(BaseInstrumentor):wrap_function_wrapper("gpt4all","Embed4All.embed",- embed("gpt4all.embed", version, environment, application_name,+ embed(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.pyindex 5101bca5..f728ba34 100644--- a/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py+++ b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py@@ -1,207 +1,324 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment"""Module for monitoring GPT4All API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import handle_exception, general_tokens+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ handle_exception,+ general_tokens,+ create_metrics_attributes,+ set_server_address_and_port,+ calculate_tbt,+ calculate_ttft+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def generate(gen_ai_endpoint, version, environment, application_name,+def generate(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for generate to collect metrics.+ Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the GPT4All API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of GPT4All usage.+ pricing_info: Information used for calculating GPT4All usage.trace_content: Flag indicating whether to trace the actual content.Returns:- A function that wraps the generate method to add telemetry.+ A function that wraps the chat completions method to add telemetry."""+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'gpt4all.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ request_model,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port+ self._request_model = request_model++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ self._llmresponse += chunk+ return chunk+ except StopIteration:+ # Handling exception ensure LLM observability without disrupting operation+ try:+ self._end_time = time.time()++ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ prompt = self._kwargs.get("prompt") or self._args[0] or ""++ # Calculate tokens using input prompt and aggregated response+ input_tokens = general_tokens(prompt)+ output_tokens = general_tokens(self._llmresponse)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("repeat_penalty", 1.18))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", 200))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temp", 0.7))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 0.4))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ self._kwargs.get("top_k", 40))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ 0)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )++ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ request_model=self._request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._request_model,+ )++ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(0, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'generate' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors+ Wraps the 'chat.completions' API call to add telemetry.++ This collects metrics such as execution time, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:- wrapped: The original 'generate' method to be wrapped.+ wrapped: The original 'chat.completions' method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'generate' method.- kwargs: Keyword arguments for the 'generate' method.+ args: Positional arguments for the 'chat.completions' method.+ kwargs: Keyword arguments for the 'chat.completions' method.Returns:- The response from the original 'generate' method.+ The response from the original 'chat.completions' method."""# Check if streaming is enabled for the API callstreaming = kwargs.get("streaming", False)+ server_address, server_port = set_server_address_and_port(instance, "localhost", 80)+ request_model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"+# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- # Collect aggregated response from events- llmresponse += chunk-- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- # Calculate cost of the operation- cost = 0-- # pylint: disable=line-too-long- model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"- prompt = kwargs.get("prompt") or args[0] or ""-- # Calculate cost of the operation- cost = 0- prompt_tokens = general_tokens(prompt)- completion_tokens = general_tokens(llmresponse)- total_tokens = prompt_tokens + completion_tokens-- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", 40))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 0.4))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", 200))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 1.18))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)++ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port, request_model)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()- # pylint: disable=line-too-long- model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"- prompt = kwargs.get("prompt") or args[0] or ""+ try:+ prompt = kwargs.get("prompt") or args[0] or ""- # Calculate cost of the operation- cost = 0- prompt_tokens = general_tokens(prompt)- completion_tokens = general_tokens(response)- total_tokens = prompt_tokens + completion_tokens+ # Calculate tokens using input prompt and aggregated response+ input_tokens = general_tokens(str(prompt))+ output_tokens = general_tokens(str(response))- try:- # Set base span attribues+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- kwargs.get("top_k", 40))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 0.4))+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("repeat_penalty", 1.18))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,kwargs.get("max_tokens", 200))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ kwargs.get("presence_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.7))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 1.18))+ kwargs.get("temp", 0.7))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ kwargs.get("top_k", 40))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ if isinstance(response, str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ 0)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -216,38 +333,33 @@ def generate(gen_ai_endpoint, version, environment, application_name,},)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(total_tokens, attributes)- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(0, attributes)# Return original responsereturn response@@ -261,20 +373,19 @@ def generate(gen_ai_endpoint, version, environment, application_name,return wrapper-def embed(gen_ai_endpoint, version, environment, application_name,- tracer, pricing_info, trace_content, metrics, disable_metrics):+def embed(version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the GPT4All API.tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of GPT4All usage.+ pricing_info: Information used for calculating GPT4All usage.trace_content: Flag indicating whether to trace the actual content.-+Returns:A function that wraps the embeddings method to add telemetry."""@@ -283,7 +394,7 @@ def embed(gen_ai_endpoint, version, environment, application_name,"""Wraps the 'embeddings' API call to add telemetry.- This collects metrics such as execution time, cost, and token usage, and handles errors+ This collects metrics such as execution time, and token usage, and handles errorsgracefully, adding details to the trace for observability.Args:@@ -296,68 +407,82 @@ def embed(gen_ai_endpoint, version, environment, application_name,The response from the original 'embeddings' method."""- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ server_address, server_port = set_server_address_and_port(instance, "localhost", 80)++ # pylint: disable=line-too-long+ request_model = str(instance.gpt4all.model.model_path).rsplit('/', maxsplit=1)[-1] or "all-MiniLM-L6-v2.gguf2.f16.gguf"++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"++ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()try:- # pylint: disable=line-too-long- model = str(instance.gpt4all.model.model_path).rsplit('/', maxsplit=1)[-1] or "all-MiniLM-L6-v2.gguf2.f16.gguf"prompt = kwargs.get("prompt") or args[0] or ""+ input_tokens = general_tokens(prompt)- # Calculate cost of the operation- cost = 0- prompt_tokens = general_tokens(prompt)-- # Set Span attributes+ # Set Span attributes (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- model)+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ request_model)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)+ input_tokens)++ # Set Span attributes (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens)+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ 0)+if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: str(kwargs.get("input", "")),},)span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- model- }-+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_EMBEDDING,+ system=SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=request_model,+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(prompt_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(0, attributes)+# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/groq/__init__.py b/sdk/python/src/openlit/instrumentation/groq/__init__.pyindex 69148a71..091b154f 100644--- a/sdk/python/src/openlit/instrumentation/groq/__init__.py+++ b/sdk/python/src/openlit/instrumentation/groq/__init__.py@@ -33,7 +33,7 @@ class GroqInstrumentor(BaseInstrumentor):wrap_function_wrapper("groq.resources.chat.completions","Completions.create",- chat("groq.chat.completions", version, environment, application_name,+ chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)@@ -41,7 +41,7 @@ class GroqInstrumentor(BaseInstrumentor):wrap_function_wrapper("groq.resources.chat.completions","AsyncCompletions.create",- async_chat("groq.chat.completions", version, environment, application_name,+ async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics),)diff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.pyindex 1033d418..473a8cd5 100644--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py@@ -1,24 +1,31 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, used-before-assignment, too-many-branches"""Module for monitoring Groq API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_chat_model_cost, handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_chat(gen_ai_endpoint, version, environment, application_name,+def async_chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Groq API.@@ -30,10 +37,245 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,A function that wraps the chat completions method to add telemetry."""+ class TracedAsyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'Groq.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._system_fingerprint = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ # Collect message IDs and aggregated response from events+ if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and+ 'content' in chunked.get('choices')[0].get('delta'))):++ content = chunked.get('choices')[0].get('delta').get('content')+ if content:+ self._llmresponse += content++ if chunked.get('usage'):+ self._input_tokens = chunked.get('usage').get('prompt_tokens')+ self._output_tokens = chunked.get('usage').get('completion_tokens')+ self._response_id = chunked.get('id')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')+ self._system_fingerprint = chunked.get('system_fingerprint')+ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GROQ)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_completion_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,+ self._kwargs.get("service_tier", "on_demand"))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,+ self._system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,+ self._kwargs.get("user", ""))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GROQ,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+async def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'chat.completions' API call to add telemetry.-+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.@@ -49,148 +291,27 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.groq.com", 443)+ request_model = kwargs.get("model", "gpt-4o")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- # Collect message IDs and aggregated response from events- if len(chunk.choices) > 0:- # pylint: disable=line-too-long- if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):- content = chunk.choices[0].delta.content- if content:- llmresponse += content- if chunk.x_groq is not None and chunk.x_groq.usage is not None:- prompt_tokens = chunk.x_groq.usage.prompt_tokens- completion_tokens = chunk.x_groq.usage.completion_tokens- total_tokens = chunk.x_groq.usage.total_tokens- response_id = chunk.x_groq.id- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GROQ)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "gpt-3.5-turbo"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GROQ,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "gpt-3.5-turbo")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return stream_generator()+ return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = await wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string@@ -202,7 +323,6 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -212,38 +332,70 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GROQ)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.x_groq["id"])- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GROQ)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3-8b-8192"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("name", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))+ kwargs.get("max_completion_tokens", -1))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,+ kwargs.get("service_tier", "on_demand"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,+ response_dict.get('system_fingerprint'))++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,+ kwargs.get("user", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -252,90 +404,54 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,},)- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3-8b-8192"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ for i in range(kwargs.get('n',1)):span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0].finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content,- },- )-- # Set span attributes for when n > 0- else:- i = 0- while i < kwargs["n"] and trace_content is True:- attribute_name = f"gen_ai.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[i].message.content,- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,- "Function called with tools")- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GROQ,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "gpt-3.5-turbo")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GROQ,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/groq/groq.py b/sdk/python/src/openlit/instrumentation/groq/groq.pyindex df89f15d..68bdc8ea 100644--- a/sdk/python/src/openlit/instrumentation/groq/groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/groq.py@@ -1,24 +1,31 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, used-before-assignment, too-many-branches"""Module for monitoring Groq API calls."""import logging+import timefrom opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import get_chat_model_cost, handle_exception+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from openlit.__helpers import (+ get_chat_model_cost,+ handle_exception,+ response_as_dict,+ calculate_ttft,+ calculate_tbt,+ create_metrics_attributes,+ set_server_address_and_port+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def chat(gen_ai_endpoint, version, environment, application_name,+def chat(version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for chat completions to collect metrics.Args:- gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.environment: Deployment environment (e.g., production, staging).application_name: Name of the application using the Groq API.@@ -30,10 +37,245 @@ def chat(gen_ai_endpoint, version, environment, application_name,A function that wraps the chat completions method to add telemetry."""+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'Groq.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._system_fingerprint = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ end_time = time.time()+ # Record the timestamp for the current chunk+ self._timestamps.append(end_time)++ if len(self._timestamps) == 1:+ # Calculate time to first chunk+ self._ttft = calculate_ttft(self._timestamps, self._start_time)++ chunked = response_as_dict(chunk)+ # Collect message IDs and aggregated response from events+ if (len(chunked.get('choices')) > 0 and ('delta' in chunked.get('choices')[0] and+ 'content' in chunked.get('choices')[0].get('delta'))):++ content = chunked.get('choices')[0].get('delta').get('content')+ if content:+ self._llmresponse += content++ if chunked.get('usage'):+ self._input_tokens = chunked.get('usage').get('prompt_tokens')+ self._output_tokens = chunked.get('usage').get('completion_tokens')+ self._response_id = chunked.get('id')+ self._response_model = chunked.get('model')+ self._finish_reason = chunked.get('choices')[0].get('finish_reason')+ self._system_fingerprint = chunked.get('system_fingerprint')+ return chunk+ except StopIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ self._end_time = time.time()+ if len(self._timestamps) > 1:+ self._tbt = calculate_tbt(self._timestamps)++ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str_list = []+ for item in content:+ if item["type"] == "text":+ content_str_list.append(f'text: {item["text"]}')+ elif (item["type"] == "image_url" and+ not item["image_url"]["url"].startswith("data:")):+ content_str_list.append(f'image_url: {item["image_url"]["url"]}')+ content_str = ", ".join(content_str_list)+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ request_model = self._kwargs.get("model", "gpt-4o")++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, self._input_tokens,+ self._output_tokens)++ # Set Span attributes (OTel Semconv)+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,+ SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GROQ)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ request_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.SERVER_PORT,+ self._server_port)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_completion_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ self._kwargs.get("stop", []))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ [self._finish_reason])+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ self._response_model)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ self._input_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ self._output_tokens)+ self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ self._server_address)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,+ self._kwargs.get("service_tier", "on_demand"))+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,+ self._system_fingerprint)+ if isinstance(self._llmresponse, str):+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ else:+ self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")++ # Set Span attributes (Extra)+ self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ self._span.set_attribute(SERVICE_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,+ self._kwargs.get("user", ""))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ self._input_tokens + self._output_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TBT,+ self._tbt)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ self._ttft)+ self._span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )+ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GROQ,+ request_model=request_model,+ server_address=self._server_address,+ server_port=self._server_port,+ response_model=self._response_model,+ )++ metrics["genai_client_usage_tokens"].record(+ self._input_tokens + self._output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ self._end_time - self._start_time, attributes+ )+ metrics["genai_server_tbt"].record(+ self._tbt, attributes+ )+ metrics["genai_server_ttft"].record(+ self._ttft, attributes+ )+ metrics["genai_requests"].add(1, attributes)+ metrics["genai_completion_tokens"].add(self._output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'chat.completions' API call to add telemetry.-+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.@@ -49,148 +291,27 @@ def chat(gen_ai_endpoint, version, environment, application_name,# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)+ server_address, server_port = set_server_address_and_port(instance, "api.groq.com", 443)+ request_model = kwargs.get("model", "gpt-4o")++ span_name = f"{SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- for chunk in wrapped(*args, **kwargs):- # Collect message IDs and aggregated response from events- if len(chunk.choices) > 0:- # pylint: disable=line-too-long- if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):- content = chunk.choices[0].delta.content- if content:- llmresponse += content- if chunk.x_groq is not None and chunk.x_groq.usage is not None:- prompt_tokens = chunk.x_groq.usage.prompt_tokens- completion_tokens = chunk.x_groq.usage.completion_tokens- total_tokens = chunk.x_groq.usage.total_tokens- response_id = chunk.x_groq.id- yield chunk-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GROQ)- span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "gpt-3.5-turbo"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GROQ,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "gpt-3.5-turbo")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- total_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return stream_generator()+ return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)# Handling for non-streaming responseselse:- # pylint: disable=line-too-long- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ start_time = time.time()response = wrapped(*args, **kwargs)+ end_time = time.time()++ response_dict = response_as_dict(response)try:# Format 'messages' into a single string@@ -202,7 +323,6 @@ def chat(gen_ai_endpoint, version, environment, application_name,if isinstance(content, list):content_str = ", ".join(- # pylint: disable=line-too-longf'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'if "type" in item else f'text: {item["text"]}'for item in content@@ -212,38 +332,70 @@ def chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Set base span attribues+ input_tokens = response_dict.get('usage').get('prompt_tokens')+ output_tokens = response_dict.get('usage').get('completion_tokens')++ # Calculate cost of the operation+ cost = get_chat_model_cost(request_model,+ pricing_info, input_tokens,+ output_tokens)++ # Set base span attribues (OTel Semconv)span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_GROQ)span.set_attribute(SemanticConvetion.GEN_AI_OPERATION,SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response.x_groq["id"])- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GROQ)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "llama3-8b-8192"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))+ request_model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.SERVER_PORT,+ server_port)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 0.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("name", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))+ kwargs.get("max_completion_tokens", -1))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_STOP_SEQUENCES,+ kwargs.get("stop", []))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 1.0))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ response_dict.get("id"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_MODEL,+ response_dict.get('model'))+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.SERVER_ADDRESS,+ server_address)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,+ kwargs.get("service_tier", "on_demand"))+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,+ response_dict.get('system_fingerprint'))++ # Set base span attribues (Extras)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT,+ environment)+ span.set_attribute(SERVICE_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,+ kwargs.get("user", ""))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ span.set_attribute(SemanticConvetion.GEN_AI_SERVER_TTFT,+ end_time - start_time)+ span.set_attribute(SemanticConvetion.GEN_AI_SDK_VERSION,+ version)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -252,90 +404,54 @@ def chat(gen_ai_endpoint, version, environment, application_name,},)- # Set span attributes when tools is not passed to the function call- if "tools" not in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "llama3-8b-8192"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)+ for i in range(kwargs.get('n',1)):span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,- [response.choices[0].finish_reason])- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)-- # Set span attributes for when n = 1 (default)- if "n" not in kwargs or kwargs["n"] == 1:- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[0].message.content,- },- )-- # Set span attributes for when n > 0- else:- i = 0- while i < kwargs["n"] and trace_content is True:- attribute_name = f"gen_ai.completion.{i}"- span.add_event(- name=attribute_name,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: response.choices[i].message.content,- },- )- i += 1-- # Return original response- return response-- # Set span attributes when tools is passed to the function call- elif "tools" in kwargs:- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),- pricing_info, response.usage.prompt_tokens,- response.usage.completion_tokens)-- span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,- "Function called with tools")- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_INPUT_TOKENS,- response.usage.prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_OUTPUT_TOKENS,- response.usage.completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- response.usage.total_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)+ [response_dict.get('choices')[i].get('finish_reason')])+ if trace_content:+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ # pylint: disable=line-too-long+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: str(response_dict.get('choices')[i].get('message').get('content')),+ },+ )+ if kwargs.get('tools'):+ span.set_attribute(SemanticConvetion.GEN_AI_TOOL_CALLS,+ str(response_dict.get('choices')[i].get('message').get('tool_calls')))++ if isinstance(response_dict.get('choices')[i].get('message').get('content'), str):+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "text")+ elif response_dict.get('choices')[i].get('message').get('content') is not None:+ span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,+ "json")span.set_status(Status(StatusCode.OK))if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_GROQ,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_OPERATION:- SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "gpt-3.5-turbo")- }+ attributes = create_metrics_attributes(+ service_name=application_name,+ deployment_environment=environment,+ operation=SemanticConvetion.GEN_AI_OPERATION_TYPE_CHAT,+ system=SemanticConvetion.GEN_AI_SYSTEM_GROQ,+ request_model=request_model,+ server_address=server_address,+ server_port=server_port,+ response_model=response_dict.get('model'),+ )+ metrics["genai_client_usage_tokens"].record(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_client_operation_duration"].record(+ end_time - start_time, attributes+ )+ metrics["genai_server_ttft"].record(+ end_time - start_time, attributes+ )metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)- metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)metrics["genai_cost"].record(cost, attributes)# Return original responsediff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.pyindex 5d1acde7..bd25bdc5 100644--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py@@ -197,11 +197,11 @@ def async_chat_completions(version, environment, application_name,output_tokens)self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_REQUEST_SERVICE_TIER,+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,self._kwargs.get("service_tier", "auto"))- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SERVICE_TIER,self._openai_response_service_tier)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,self._openai_system_fingerprint)if isinstance(self._llmresponse, str):self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,@@ -382,11 +382,11 @@ def async_chat_completions(version, environment, application_name,output_tokens)span.set_attribute(SemanticConvetion.SERVER_ADDRESS,server_address)- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_REQUEST_SERVICE_TIER,+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,kwargs.get("service_tier", "auto"))- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SERVICE_TIER,response_dict.get('service_tier'))- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,response_dict.get('system_fingerprint'))# Set base span attribues (Extras)diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.pyindex 7c39adb2..95378d88 100644--- a/sdk/python/src/openlit/instrumentation/openai/openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py@@ -197,11 +197,11 @@ def chat_completions(version, environment, application_name,output_tokens)self._span.set_attribute(SemanticConvetion.SERVER_ADDRESS,self._server_address)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_REQUEST_SERVICE_TIER,+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,self._kwargs.get("service_tier", "auto"))- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SERVICE_TIER,self._openai_response_service_tier)- self._span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,self._openai_system_fingerprint)if isinstance(self._llmresponse, str):self._span.set_attribute(SemanticConvetion.GEN_AI_OUTPUT_TYPE,@@ -382,11 +382,11 @@ def chat_completions(version, environment, application_name,output_tokens)span.set_attribute(SemanticConvetion.SERVER_ADDRESS,server_address)- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_REQUEST_SERVICE_TIER,+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SERVICE_TIER,kwargs.get("service_tier", "auto"))- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SERVICE_TIER,response_dict.get('service_tier'))- span.set_attribute(SemanticConvetion.GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT,+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT,response_dict.get('system_fingerprint'))# Set base span attribues (Extras)diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 616722a5..84abc899 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -88,9 +88,9 @@ class SemanticConvetion:GEN_AI_SYSTEM_XAI = "xai"# GenAI OpenAI Attributes (OTel Semconv)- GEN_AI_OPENAI_REQUEST_SERVICE_TIER = "gen_ai.openai.request.service_tier"- GEN_AI_OPENAI_RESPONSE_SERVICE_TIER = "gen_ai.openai.response.service_tier"- GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT = "gen_ai.openai.response.system_fingerprint"+ GEN_AI_REQUEST_SERVICE_TIER = "gen_ai.request.service_tier"+ GEN_AI_RESPONSE_SERVICE_TIER = "gen_ai.response.service_tier"+ GEN_AI_RESPONSE_SYSTEM_FINGERPRINT = "gen_ai.response.system_fingerprint"# GenAI System Names (Extra)GEN_AI_SYSTEM_HUGGING_FACE = "huggingface"