Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[FIX]: asynchronous context manager wrapper around openai.AsyncStream (#494)
Mohanad Abugharbiaca7bc5152ad8
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex efbd958a..d5017db8 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.29.1"+version = "1.29.2"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.pyindex 2768ea22..e3118464 100644--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py@@ -31,10 +31,179 @@ def async_chat_completions(gen_ai_endpoint, version, environment, application_naA function that wraps the chat completions method to add telemetry."""+ class TracedAsyncStream:+ """+ Wrapper for streaming responses to collect metrics and trace data.+ Wraps the 'openai.AsyncStream' response to collect message IDs and aggregated response.++ This class implements the '__aiter__' and '__anext__' methods that+ handle asynchronous streaming responses.++ This class also implements '__aenter__' and '__aexit__' methods that+ handle asynchronous context management protocol.+ """+ def __init__(+ self,+ wrapped,+ span,+ *args,+ **kwargs,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ # Placeholder for aggregating streaming response+ self._llmresponse = ""+ self._response_id = ""++ self._args = args+ self._kwargs = kwargs++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ # Collect message IDs and aggregated response from events+ if len(chunk.choices) > 0:+ # pylint: disable=line-too-long+ if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):+ content = chunk.choices[0].delta.content+ if content:+ self._llmresponse += content+ self._response_id = chunk.id+ return chunk+ except StopAsyncIteration:+ # Handling exception ensure observability without disrupting operation+ try:+ # Format 'messages' into a single string+ message_prompt = self._kwargs.get("messages", "")+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str = ", ".join(+ # pylint: disable=line-too-long+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)++ # Calculate tokens using input prompt and aggregated response+ prompt_tokens = openai_tokens(prompt,+ self._kwargs.get("model", "gpt-3.5-turbo"))+ completion_tokens = openai_tokens(self._llmresponse,+ self._kwargs.get("model", "gpt-3.5-turbo"))++ # Calculate cost of the operation+ cost = get_chat_model_cost(self._kwargs.get("model", "gpt-3.5-turbo"),+ pricing_info, prompt_tokens,+ completion_tokens)++ # Set Span attributes+ self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ self._span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)+ self._span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_CHAT)+ self._span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ self._span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,+ self._response_id)+ self._span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ self._kwargs.get("model", "gpt-3.5-turbo"))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,+ self._kwargs.get("user", ""))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ self._kwargs.get("top_p", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ self._kwargs.get("max_tokens", -1))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ self._kwargs.get("temperature", 1.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,+ self._kwargs.get("presence_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ self._kwargs.get("frequency_penalty", 0.0))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,+ self._kwargs.get("seed", ""))+ self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens + completion_tokens)+ self._span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ if trace_content:+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ self._span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: self._llmresponse,+ },+ )++ self._span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ self._kwargs.get("model", "gpt-3.5-turbo")+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(+ prompt_tokens + completion_tokens, attributes+ )+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(self._span, e)+ logger.error("Error in trace creation: %s", e)+ finally:+ self._span.end()+ raise+async def wrapper(wrapped, instance, args, kwargs):"""Wraps the 'chat.completions' API call to add telemetry.-+This collects metrics such as execution time, cost, and token usage, and handles errorsgracefully, adding details to the trace for observability.@@ -54,140 +223,10 @@ def async_chat_completions(gen_ai_endpoint, version, environment, application_na# pylint: disable=no-else-returnif streaming:# Special handling for streaming response to accommodate the nature of data flow- async def stream_generator():- with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:- # Placeholder for aggregating streaming response- llmresponse = ""-- # Loop through streaming events capturing relevant details- async for chunk in await wrapped(*args, **kwargs):- # Collect message IDs and aggregated response from events- if len(chunk.choices) > 0:- # pylint: disable=line-too-long- if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):- content = chunk.choices[0].delta.content- if content:- llmresponse += content- yield chunk- response_id = chunk.id-- # Handling exception ensure observability without disrupting operation- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- # pylint: disable=line-too-long- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- # Calculate tokens using input prompt and aggregated response- prompt_tokens = openai_tokens(prompt,- kwargs.get("model", "gpt-3.5-turbo"))- completion_tokens = openai_tokens(llmresponse,- kwargs.get("model", "gpt-3.5-turbo"))+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(gen_ai_endpoint, kind=SpanKind.CLIENT)- # Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),- pricing_info, prompt_tokens,- completion_tokens)-- # Set Span attributes- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,- SemanticConvetion.GEN_AI_SYSTEM_OPENAI)- span.set_attribute(SemanticConvetion.GEN_AI_TYPE,- SemanticConvetion.GEN_AI_TYPE_CHAT)- span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,- gen_ai_endpoint)- span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,- response_id)- span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,- environment)- span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,- application_name)- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "gpt-3.5-turbo"))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,- kwargs.get("user", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- kwargs.get("top_p", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 1.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,- True)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,- prompt_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,- completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- prompt_tokens + completion_tokens)- span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- cost)- if trace_content:- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llmresponse,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = {- TELEMETRY_SDK_NAME:- "openlit",- SemanticConvetion.GEN_AI_APPLICATION_NAME:- application_name,- SemanticConvetion.GEN_AI_SYSTEM:- SemanticConvetion.GEN_AI_SYSTEM_OPENAI,- SemanticConvetion.GEN_AI_ENVIRONMENT:- environment,- SemanticConvetion.GEN_AI_TYPE:- SemanticConvetion.GEN_AI_TYPE_CHAT,- SemanticConvetion.GEN_AI_REQUEST_MODEL:- kwargs.get("model", "gpt-3.5-turbo")- }-- metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(- prompt_tokens + completion_tokens, attributes- )- metrics["genai_completion_tokens"].add(completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)-- return stream_generator()+ return TracedAsyncStream(awaited_wrapped, span)# Handling for non-streaming responseselse:@@ -363,7 +402,7 @@ def async_embedding(gen_ai_endpoint, version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for embeddings to collect metrics.-+Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.@@ -372,7 +411,7 @@ def async_embedding(gen_ai_endpoint, version, environment, application_name,tracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of OpenAI usage.trace_content: Flag indicating whether to trace the actual content.-+Returns:A function that wraps the embeddings method to add telemetry."""@@ -475,7 +514,7 @@ def async_finetune(gen_ai_endpoint, version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for fine-tuning jobs to collect metrics.-+Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.@@ -484,7 +523,7 @@ def async_finetune(gen_ai_endpoint, version, environment, application_name,tracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of OpenAI usage.trace_content: Flag indicating whether to trace the actual content.-+Returns:A function that wraps the fine tuning creation method to add telemetry."""@@ -564,7 +603,7 @@ def async_image_generate(gen_ai_endpoint, version, environment, application_nametracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for image generation to collect metrics.-+Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.@@ -573,7 +612,7 @@ def async_image_generate(gen_ai_endpoint, version, environment, application_nametracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of OpenAI image generation.trace_content: Flag indicating whether to trace the input prompt and generated images.-+Returns:A function that wraps the image generation method to add telemetry."""@@ -694,7 +733,7 @@ def async_image_variatons(gen_ai_endpoint, version, environment, application_namtracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for creating image variations to collect metrics.-+Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.@@ -703,7 +742,7 @@ def async_image_variatons(gen_ai_endpoint, version, environment, application_namtracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of generating image variations.trace_content: Flag indicating whether to trace the input image and generated variations.-+Returns:A function that wraps the image variations creation method to add telemetry."""@@ -813,7 +852,7 @@ def async_audio_create(gen_ai_endpoint, version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""Generates a telemetry wrapper for creating speech audio to collect metrics.-+Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.version: Version of the monitoring package.@@ -822,7 +861,7 @@ def async_audio_create(gen_ai_endpoint, version, environment, application_name,tracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of generating speech audio.trace_content: Flag indicating whether to trace the input text and generated audio.-+Returns:A function that wraps the speech audio creation method to add telemetry."""