Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Refactor Cohere OpenTelemetry Monitoring Instrumentation (#781)
Patcheree0844aae46b
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex baf30a30..ecd5cbb9 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.34.17"+version = "1.34.18"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/cohere/__init__.py b/sdk/python/src/openlit/instrumentation/cohere/__init__.pyindex b83449ea..5eb79ae6 100644--- a/sdk/python/src/openlit/instrumentation/cohere/__init__.py+++ b/sdk/python/src/openlit/instrumentation/cohere/__init__.py@@ -1,74 +1,88 @@-# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code"""Initializer of Auto Instrumentation of Cohere Functions"""+from typing import Collectionimport importlib.metadatafrom opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapper-from openlit.instrumentation.cohere.cohere import chat, chat_stream, embed-from openlit.instrumentation.cohere.async_cohere import async_chat, async_chat_stream, async_embed-+from openlit.instrumentation.cohere.cohere import (+ chat,+ chat_stream,+ embed+)+from openlit.instrumentation.cohere.async_cohere import (+ async_chat,+ async_chat_stream,+ async_embed+)_instruments = ("cohere >= 5.14.0",)class CohereInstrumentor(BaseInstrumentor):- """An instrumentor for Cohere's client library."""+ """+ An instrumentor for Cohere client library.+ """def instrumentation_dependencies(self) -> Collection[str]:return _instrumentsdef _instrument(self, **kwargs):- application_name = kwargs.get("application_name")- environment = kwargs.get("environment")+ application_name = kwargs.get("application_name", "default")+ environment = kwargs.get("environment", "default")tracer = kwargs.get("tracer")metrics = kwargs.get("metrics_dict")- pricing_info = kwargs.get("pricing_info")- capture_message_content = kwargs.get("capture_message_content")+ pricing_info = kwargs.get("pricing_info", {})+ capture_message_content = kwargs.get("capture_message_content", False)disable_metrics = kwargs.get("disable_metrics")version = importlib.metadata.version("cohere")- # Sync Client+ # sync chat completionswrap_function_wrapper(- "cohere.client_v2",- "ClientV2.chat",+ "cohere.client_v2",+ "ClientV2.chat",chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)++ # sync chat streamingwrap_function_wrapper(- "cohere.client_v2",- "ClientV2.chat_stream",+ "cohere.client_v2",+ "ClientV2.chat_stream",chat_stream(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)++ # sync embeddingswrap_function_wrapper(- "cohere.client_v2",- "ClientV2.embed",+ "cohere.client_v2",+ "ClientV2.embed",embed(version, environment, application_name,tracer, pricing_info, capture_message_content, metrics, disable_metrics),)- # Async Client+ # async chat completionswrap_function_wrapper(- "cohere.client_v2",- "AsyncClientV2.chat",+ "cohere.client_v2",+ "AsyncClientV2.chat",async_chat(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)+ # async chat streamingwrap_function_wrapper(- "cohere.client_v2",- "AsyncClientV2.chat_stream",+ "cohere.client_v2",+ "AsyncClientV2.chat_stream",async_chat_stream(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics),+ tracer, pricing_info, capture_message_content, metrics, disable_metrics),)+ # async embeddingswrap_function_wrapper(- "cohere.client_v2",- "AsyncClientV2.embed",+ "cohere.client_v2",+ "AsyncClientV2.embed",async_embed(version, environment, application_name,tracer, pricing_info, capture_message_content, metrics, disable_metrics),)- @staticmethoddef _uninstrument(self, **kwargs):passdiff --git a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.pyindex b4d14189..aab3f5b2 100644--- a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py@@ -1,185 +1,145 @@"""-Module for monitoring Cohere API calls.+Module for monitoring Cohere API calls (async version)."""-import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,- get_embed_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,- set_server_address_and_port+ set_server_address_and_port,+)+from openlit.instrumentation.cohere.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,+ process_embedding_response,)from openlit.semcov import SemanticConvention-# Initialize logger for logging potential issues and operations-logger = logging.getLogger(__name__)--def async_embed(version, environment, application_name, tracer,- pricing_info, capture_message_content, metrics, disable_metrics):+def async_chat(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for embeddings to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the embeddings method to add telemetry.+ Generates a telemetry wrapper for GenAI chat function call"""async def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'embed' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'embed' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'embed' method.- kwargs: Keyword arguments for the 'embed' method.-- Returns:- The response from the original 'embed' method.+ Wraps the GenAI chat function call."""server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)- request_model = kwargs.get("model", "mbed-english-v3.0")+ request_model = kwargs.get("model", "command-r-plus-08-2024")- span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = await wrapped(*args, **kwargs)- end_time = time.time()+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ return response- response_dict = response_as_dict(response)- try:- input_tokens = response_dict.get('meta').get('billed_units').get('input_tokens')- # Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get("model", "embed-english-v2.0"),- pricing_info, input_tokens)-- # Set Span attributes (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get('embedding_types', ['float']))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- response_dict.get('response_type'))-- # Set Span attributes (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: str(kwargs.get("texts", "")),- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )- metrics["genai_client_usage_tokens"].record(- input_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ return wrapper- # Return original response- return response+def async_chat_stream(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for GenAI chat_stream function call+ """- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ class TracedAsyncStream:+ """+ Wrapper for async streaming responses to collect telemetry.+ """- # Return original response- return response+ def __init__(+ self,+ wrapped,+ span,+ span_name,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ self._span_name = span_name+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._tools = None+ self._tool_plan = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ async def __aenter__(self):+ await self.__wrapped__.__aenter__()+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)++ def __aiter__(self):+ return self++ async def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(await self.__wrapped__, name)++ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ process_chunk(self, chunk)+ return chunk+ except StopAsyncIteration:+ try:+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version+ )- return wrapper+ except Exception as e:+ handle_exception(self._span, e)-def async_chat(version, environment, application_name, tracer,- pricing_info, capture_message_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for chat to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """+ raiseasync def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.-- Returns:- The response from the original 'chat' method.+ Wraps the GenAI chat_stream function call."""server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)@@ -187,424 +147,55 @@ def async_chat(version, environment, application_name, tracer,span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:- start_time = time.time()- response = await wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('billed_units').get('input_tokens')- output_tokens = response_dict.get('usage').get('billed_units').get('output_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model, pricing_info,- input_tokens, output_tokens)-- llm_response = response_dict.get('message').get('content')[0].get('text')-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop_sequences", []))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.3))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K,- kwargs.get("k", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- kwargs.get("p", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- if isinstance(llm_response, str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: llm_response,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # Stream endpoint is always streaming+ awaited_wrapped = await wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- # Return original response- return response+ return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)return wrapper-def async_chat_stream(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+def async_embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat_stream to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for GenAI embedding function call"""async def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat_stream' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat_stream' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat_stream' method.- kwargs: Keyword arguments for the 'chat_stream' method.-- Returns:- The response from the original 'chat_stream' method.+ Wraps the GenAI embedding function call."""- class TracedAsyncStream:- """- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.- """- def __init__(- self,- wrapped,- span,- kwargs,- server_address,- server_port,- **args,- ):- self.__wrapped__ = wrapped- self._span = span- # Placeholder for aggregating streaming response- self._llmresponse = ""- self._response_id = ""- self._finish_reason = ""- self._input_tokens = ""- self._output_tokens = ""-- self._args = args- self._kwargs = kwargs- self._start_time = time.time()- self._end_time = None- self._timestamps = []- self._ttft = 0- self._tbt = 0- self._server_address = server_address- self._server_port = server_port-- async def __aenter__(self):- await self.__wrapped__.__aenter__()- return self-- async def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)-- def __aiter__(self):- return self-- async def __getattr__(self, name):- """Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)-- async def __anext__(self):- try:- chunk = await self.__wrapped__.__anext__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)-- if chunked.get('type') == 'message-start':- self._response_id = chunked.get('id')-- if chunked.get('type') == 'content-delta':- content = chunked.get('delta').get('message').get('text')- if content:- self._llmresponse += content-- if chunked.get('type') == 'message-end':- self._finish_reason = chunked.get('delta').get('finish_reason')- self._input_tokens = chunked.get('delta').get('usage').get('billed_units').get('input_tokens')- self._output_tokens = chunked.get('delta').get('usage').get('billed_units').get('output_tokens')-- return chunk- except StopAsyncIteration:- # Handling exception ensure observability without disrupting operation- try:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "command-r-plus")-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_tokens", -1))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop_sequences", []))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 0.3))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K,- self._kwargs.get("k", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("p", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)-- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()- raise-server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)- request_model = kwargs.get("model", "command-r-plus")+ request_model = kwargs.get("model", "embed-english-v3.0")- span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"- awaited_wrapped = await wrapped(*args, **kwargs)- span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedAsyncStream(awaited_wrapped, span, kwargs, server_address, server_port)+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = await wrapped(*args, **kwargs)++ try:+ response = process_embedding_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ except Exception as e:+ handle_exception(span, e)++ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/cohere/cohere.py b/sdk/python/src/openlit/instrumentation/cohere/cohere.pyindex fc98ea85..d74d27fb 100644--- a/sdk/python/src/openlit/instrumentation/cohere/cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/cohere.py@@ -2,184 +2,144 @@Module for monitoring Cohere API calls."""-import loggingimport time-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT+from opentelemetry.trace import SpanKindfrom openlit.__helpers import (- get_chat_model_cost,- get_embed_model_cost,handle_exception,- response_as_dict,- calculate_ttft,- calculate_tbt,- create_metrics_attributes,- set_server_address_and_port+ set_server_address_and_port,+)+from openlit.instrumentation.cohere.utils import (+ process_chunk,+ process_chat_response,+ process_streaming_chat_response,+ process_embedding_response,)from openlit.semcov import SemanticConvention-# Initialize logger for logging potential issues and operations-logger = logging.getLogger(__name__)--def embed(version, environment, application_name, tracer,- pricing_info, capture_message_content, metrics, disable_metrics):+def chat(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for embeddings to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the embeddings method to add telemetry.+ Generates a telemetry wrapper for GenAI chat function call"""def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'embed' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'embed' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'embed' method.- kwargs: Keyword arguments for the 'embed' method.-- Returns:- The response from the original 'embed' method.+ Wraps the GenAI chat function call."""server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)- request_model = kwargs.get("model", "mbed-english-v3.0")+ request_model = kwargs.get("model", "command-r-plus-08-2024")- span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- with tracer.start_as_current_span(span_name, kind= SpanKind.CLIENT) as span:+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:start_time = time.time()response = wrapped(*args, **kwargs)- end_time = time.time()+ response = process_chat_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ return response- response_dict = response_as_dict(response)- try:- input_tokens = response_dict.get('meta').get('billed_units').get('input_tokens')- # Calculate cost of the operation- cost = get_embed_model_cost(kwargs.get("model", "embed-english-v2.0"),- pricing_info, input_tokens)-- # Set Span attributes (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS,- kwargs.get('embedding_types', ['float']))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- response_dict.get('response_type'))-- # Set Span attributes (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: str(kwargs.get("texts", "")),- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )- metrics["genai_client_usage_tokens"].record(- input_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)+ return wrapper- # Return original response- return response+def chat_stream(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for GenAI chat_stream function call+ """- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ class TracedSyncStream:+ """+ Wrapper for streaming responses to collect telemetry.+ """- # Return original response- return response+ def __init__(+ self,+ wrapped,+ span,+ span_name,+ kwargs,+ server_address,+ server_port,+ **args,+ ):+ self.__wrapped__ = wrapped+ self._span = span+ self._span_name = span_name+ self._llmresponse = ""+ self._response_id = ""+ self._response_model = ""+ self._finish_reason = ""+ self._tools = None+ self._tool_plan = ""+ self._input_tokens = 0+ self._output_tokens = 0++ self._args = args+ self._kwargs = kwargs+ self._start_time = time.time()+ self._end_time = None+ self._timestamps = []+ self._ttft = 0+ self._tbt = 0+ self._server_address = server_address+ self._server_port = server_port++ def __enter__(self):+ self.__wrapped__.__enter__()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)++ def __iter__(self):+ return self++ def __getattr__(self, name):+ """Delegate attribute access to the wrapped object."""+ return getattr(self.__wrapped__, name)++ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ process_chunk(self, chunk)+ return chunk+ except StopIteration:+ try:+ with tracer.start_as_current_span(self._span_name, kind= SpanKind.CLIENT) as self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version+ )- return wrapper+ except Exception as e:+ handle_exception(self._span, e)-def chat(version, environment, application_name, tracer,- pricing_info, capture_message_content, metrics, disable_metrics):- """- Generates a telemetry wrapper for chat to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.- """+ raisedef wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat' method.- kwargs: Keyword arguments for the 'chat' method.-- Returns:- The response from the original 'chat' method.+ Wraps the GenAI chat_stream function call."""server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)@@ -187,424 +147,55 @@ def chat(version, environment, application_name, tracer,span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"- with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:- start_time = time.time()- response = wrapped(*args, **kwargs)- end_time = time.time()-- response_dict = response_as_dict(response)-- try:- # Format 'messages' into a single string- message_prompt = kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str = ", ".join(- f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'- if "type" in item else f'text: {item["text"]}'- for item in content- )- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- input_tokens = response_dict.get('usage').get('billed_units').get('input_tokens')- output_tokens = response_dict.get('usage').get('billed_units').get('output_tokens')-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model, pricing_info,- input_tokens, output_tokens)-- llm_response = response_dict.get('message').get('content')[0].get('text')-- # Set base span attribues (OTel Semconv)- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- kwargs.get("seed", ""))- span.set_attribute(SemanticConvention.SERVER_PORT,- server_port)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- kwargs.get("frequency_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- kwargs.get("max_tokens", -1))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- kwargs.get("presence_penalty", 0.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- kwargs.get("stop_sequences", []))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- kwargs.get("temperature", 0.3))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K,- kwargs.get("k", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- kwargs.get("p", 1.0))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- response_dict.get("id"))- span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- input_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- output_tokens)- span.set_attribute(SemanticConvention.SERVER_ADDRESS,- server_address)- if isinstance(llm_response, str):- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set base span attribues (Extras)- span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- span.set_attribute(SERVICE_NAME,- application_name)- span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- False)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- input_tokens + output_tokens)- span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- end_time - start_time)- span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)-- if capture_message_content:- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: llm_response,- },- )-- span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=server_address,- server_port=server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- input_tokens + output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- end_time - start_time, attributes- )- metrics["genai_server_ttft"].record(- end_time - start_time, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(output_tokens, attributes)- metrics["genai_prompt_tokens"].add(input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- # Return original response- return response-- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # Stream endpoint is always streaming+ awaited_wrapped = wrapped(*args, **kwargs)+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- # Return original response- return response+ return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)return wrapper-def chat_stream(version, environment, application_name,- tracer, pricing_info, capture_message_content, metrics, disable_metrics):+def embed(version, environment, application_name,+ tracer, pricing_info, capture_message_content, metrics, disable_metrics):"""- Generates a telemetry wrapper for chat_stream to collect metrics.-- Args:- version: Version of the monitoring package.- environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Cohere API.- tracer: OpenTelemetry tracer for creating spans.- pricing_info: Information used for calculating the cost of Cohere usage.- capture_message_content: Flag indicating whether to trace the actual content.-- Returns:- A function that wraps the chat method to add telemetry.+ Generates a telemetry wrapper for GenAI embedding function call"""def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat_stream' API call to add telemetry.-- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.-- Args:- wrapped: The original 'chat_stream' method to be wrapped.- instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat_stream' method.- kwargs: Keyword arguments for the 'chat_stream' method.-- Returns:- The response from the original 'chat_stream' method.+ Wraps the GenAI embedding function call."""- class TracedSyncStream:- """- Wrapper for streaming responses to collect metrics and trace data.- Wraps the response to collect message IDs and aggregated response.-- This class implements the '__aiter__' and '__anext__' methods that- handle asynchronous streaming responses.-- This class also implements '__aenter__' and '__aexit__' methods that- handle asynchronous context management protocol.- """- def __init__(- self,- wrapped,- span,- kwargs,- server_address,- server_port,- **args,- ):- self.__wrapped__ = wrapped- self._span = span- # Placeholder for aggregating streaming response- self._llmresponse = ""- self._response_id = ""- self._finish_reason = ""- self._input_tokens = ""- self._output_tokens = ""-- self._args = args- self._kwargs = kwargs- self._start_time = time.time()- self._end_time = None- self._timestamps = []- self._ttft = 0- self._tbt = 0- self._server_address = server_address- self._server_port = server_port-- def __enter__(self):- self.__wrapped__.__enter__()- return self-- def __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)-- def __iter__(self):- return self-- def __getattr__(self, name):- """Delegate attribute access to the wrapped object."""- return getattr(self.__wrapped__, name)-- def __next__(self):- try:- chunk = self.__wrapped__.__next__()- end_time = time.time()- # Record the timestamp for the current chunk- self._timestamps.append(end_time)-- if len(self._timestamps) == 1:- # Calculate time to first chunk- self._ttft = calculate_ttft(self._timestamps, self._start_time)-- chunked = response_as_dict(chunk)-- if chunked.get('type') == 'message-start':- self._response_id = chunked.get('id')-- if chunked.get('type') == 'content-delta':- content = chunked.get('delta').get('message').get('text')- if content:- self._llmresponse += content-- if chunked.get('type') == 'message-end':- self._finish_reason = chunked.get('delta').get('finish_reason')- self._input_tokens = chunked.get('delta').get('usage').get('billed_units').get('input_tokens')- self._output_tokens = chunked.get('delta').get('usage').get('billed_units').get('output_tokens')-- return chunk- except StopIteration:- # Handling exception ensure observability without disrupting operation- try:- self._end_time = time.time()- if len(self._timestamps) > 1:- self._tbt = calculate_tbt(self._timestamps)-- # Format 'messages' into a single string- message_prompt = self._kwargs.get("messages", "")- formatted_messages = []- for message in message_prompt:- role = message["role"]- content = message["content"]-- if isinstance(content, list):- content_str_list = []- for item in content:- if item["type"] == "text":- content_str_list.append(f'text: {item["text"]}')- elif (item["type"] == "image_url" and- not item["image_url"]["url"].startswith("data:")):- content_str_list.append(f'image_url: {item["image_url"]["url"]}')- content_str = ", ".join(content_str_list)- formatted_messages.append(f"{role}: {content_str}")- else:- formatted_messages.append(f"{role}: {content}")- prompt = "\n".join(formatted_messages)-- request_model = self._kwargs.get("model", "command-r-plus")-- # Calculate cost of the operation- cost = get_chat_model_cost(request_model,- pricing_info, self._input_tokens,- self._output_tokens)-- # Set Span attributes (OTel Semconv)- self._span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- self._span.set_attribute(SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT)- self._span.set_attribute(SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_COHERE)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED,- self._kwargs.get("seed", ""))- self._span.set_attribute(SemanticConvention.SERVER_PORT,- self._server_port)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY,- self._kwargs.get("frequency_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS,- self._kwargs.get("max_tokens", -1))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY,- self._kwargs.get("presence_penalty", 0.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,- self._kwargs.get("stop_sequences", []))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE,- self._kwargs.get("temperature", 0.3))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K,- self._kwargs.get("k", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P,- self._kwargs.get("p", 1.0))- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON,- [self._finish_reason])- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID,- self._response_id)- self._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_MODEL,- request_model)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS,- self._input_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS,- self._output_tokens)- self._span.set_attribute(SemanticConvention.SERVER_ADDRESS,- self._server_address)-- if isinstance(self._llmresponse, str):- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "text")- else:- self._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE,- "json")-- # Set Span attributes (Extra)- self._span.set_attribute(DEPLOYMENT_ENVIRONMENT,- environment)- self._span.set_attribute(SERVICE_NAME,- application_name)- self._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM,- True)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,- self._input_tokens + self._output_tokens)- self._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST,- cost)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TBT,- self._tbt)- self._span.set_attribute(SemanticConvention.GEN_AI_SERVER_TTFT,- self._ttft)- self._span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION,- version)- if capture_message_content:- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,- },- )- self._span.add_event(- name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,- attributes={- SemanticConvention.GEN_AI_CONTENT_COMPLETION: self._llmresponse,- },- )- self._span.set_status(Status(StatusCode.OK))-- if disable_metrics is False:- attributes = create_metrics_attributes(- service_name=application_name,- deployment_environment=environment,- operation=SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,- system=SemanticConvention.GEN_AI_SYSTEM_COHERE,- request_model=request_model,- server_address=self._server_address,- server_port=self._server_port,- response_model=request_model,- )-- metrics["genai_client_usage_tokens"].record(- self._input_tokens + self._output_tokens, attributes- )- metrics["genai_client_operation_duration"].record(- self._end_time - self._start_time, attributes- )- metrics["genai_server_tbt"].record(- self._tbt, attributes- )- metrics["genai_server_ttft"].record(- self._ttft, attributes- )- metrics["genai_requests"].add(1, attributes)- metrics["genai_completion_tokens"].add(self._output_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._input_tokens, attributes)- metrics["genai_cost"].record(cost, attributes)-- except Exception as e:- handle_exception(self._span, e)- logger.error("Error in trace creation: %s", e)- finally:- self._span.end()- raise-server_address, server_port = set_server_address_and_port(instance, "api.cohere.com", 443)- request_model = kwargs.get("model", "command-r-plus")+ request_model = kwargs.get("model", "embed-english-v3.0")- span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING} {request_model}"- awaited_wrapped = wrapped(*args, **kwargs)- span = tracer.start_span(span_name, kind=SpanKind.CLIENT)- return TracedSyncStream(awaited_wrapped, span, kwargs, server_address, server_port)+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ response = wrapped(*args, **kwargs)++ try:+ response = process_embedding_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ **kwargs+ )++ except Exception as e:+ handle_exception(span, e)++ return responsereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/cohere/utils.py b/sdk/python/src/openlit/instrumentation/cohere/utils.pynew file mode 100644index 00000000..a368eafe--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/cohere/utils.py@@ -0,0 +1,330 @@+"""+Cohere OpenTelemetry instrumentation utility functions+"""+import time++from opentelemetry.trace import Status, StatusCode++from openlit.__helpers import (+ calculate_ttft,+ response_as_dict,+ calculate_tbt,+ get_chat_model_cost,+ get_embed_model_cost,+ common_span_attributes,+ record_completion_metrics,+ record_embedding_metrics,+)+from openlit.semcov import SemanticConvention++def format_content(messages):+ """+ Process a list of messages to extract content.+ """++ formatted_messages = []+ for message in messages:+ # Handle both dictionary and object formats+ if isinstance(message, dict):+ role = message.get("role", "user")+ content = message.get("content", "")+ else:+ # Handle Cohere object format (e.g., cohere.UserChatMessageV2)+ role = getattr(message, "role", "user")+ content = getattr(message, "content", "")++ if isinstance(content, list):+ content_str = ", ".join(+ f'{item["type"]}: {item["text"] if "text" in item else item.get("image_url", "")}'+ if "type" in item else f'text: {item.get("text", "")}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")++ return "\n".join(formatted_messages)++def process_chunk(scope, chunk):+ """+ Process a chunk of response data and update state.+ """++ end_time = time.time()+ # Record the timestamp for the current chunk+ scope._timestamps.append(end_time)++ if len(scope._timestamps) == 1:+ # Calculate time to first chunk+ scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)++ chunked = response_as_dict(chunk)++ # Handle different chunk types for Cohere streaming+ if chunked.get("type") == "message-start":+ scope._response_id = chunked.get("id")++ if chunked.get("type") == "content-delta":+ content = chunked.get("delta", {}).get("message", {}).get("content", {}).get("text")+ if content:+ scope._llmresponse += content++ # Handle tool plan deltas+ if chunked.get("type") == "tool-plan-delta":+ tool_plan_text = chunked.get("delta", {}).get("message", {}).get("tool_plan", "")+ if tool_plan_text:+ if not hasattr(scope, "_tool_plan"):+ scope._tool_plan = ""+ scope._tool_plan += tool_plan_text++ # Handle tool call start+ if chunked.get("type") == "tool-call-start":+ if not hasattr(scope, "_tools") or scope._tools is None:+ scope._tools = []++ index = chunked.get("index", 0)+ tool_call = chunked.get("delta", {}).get("message", {}).get("tool_calls", {})++ # Extend list if needed+ scope._tools.extend([{}] * (index + 1 - len(scope._tools)))++ # Initialize tool call+ scope._tools[index] = {+ "id": tool_call.get("id", ""),+ "type": tool_call.get("type", "function"),+ "function": {+ "name": tool_call.get("function", {}).get("name", ""),+ "arguments": ""+ }+ }++ # Handle tool call deltas (arguments)+ if chunked.get("type") == "tool-call-delta":+ if hasattr(scope, "_tools") and scope._tools:+ index = chunked.get("index", 0)+ if index < len(scope._tools):+ arguments = chunked.get("delta", {}).get("message", {}).get("tool_calls", {}).get("function", {}).get("arguments", "")+ if arguments:+ scope._tools[index]["function"]["arguments"] += arguments++ if chunked.get("type") == "message-end":+ delta = chunked.get("delta", {})+ scope._finish_reason = delta.get("finish_reason", "")+ usage = delta.get("usage", {}).get("billed_units", {})+ scope._input_tokens = usage.get("input_tokens", 0)+ scope._output_tokens = usage.get("output_tokens", 0)+ scope._end_time = time.time()++def common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream):+ """+ Process chat request and generate Telemetry+ """++ if len(scope._timestamps) > 1:+ scope._tbt = calculate_tbt(scope._timestamps)++ prompt = format_content(scope._kwargs.get("messages", []))+ request_model = scope._kwargs.get("model", "command-r-plus-08-2024")++ cost = get_chat_model_cost(request_model, pricing_info, scope._input_tokens, scope._output_tokens)++ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_COHERE,+ scope._server_address, scope._server_port, request_model, scope._response_model,+ environment, application_name, is_stream, scope._tbt, scope._ttft, version)++ # Span Attributes for Request parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_SEED, scope._kwargs.get("seed", ""))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, scope._kwargs.get("frequency_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, scope._kwargs.get("max_tokens", -1))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, scope._kwargs.get("presence_penalty", 0.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, scope._kwargs.get("stop_sequences", []))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, scope._kwargs.get("temperature", 0.3))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, scope._kwargs.get("k", 1.0))+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, scope._kwargs.get("p", 1.0))++ # Span Attributes for Response parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, scope._response_id)+ scope._span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason])+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "text" if isinstance(scope._llmresponse, str) else "json")++ # Span Attributes for Cost and Tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens + scope._output_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)++ # Span Attributes for Tools - optimized+ if scope._tools:+ tools = scope._tools if isinstance(scope._tools, list) else [scope._tools]++ names, ids, args = zip(*[+ (t.get("function", {}).get("name", ""),+ str(t.get("id", "")),+ str(t.get("function", {}).get("arguments", "")))+ for t in tools if isinstance(t, dict) and t+ ]) if tools else ([], [], [])++ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_NAME, ", ".join(filter(None, names)))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, ", ".join(filter(None, ids)))+ scope._span.set_attribute(SemanticConvention.GEN_AI_TOOL_ARGS, ", ".join(filter(None, args)))++ # Span Attributes for Tool Plan (Cohere specific)+ if hasattr(scope, "_tool_plan") and scope._tool_plan:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_REASONING, scope._tool_plan)++ # Span Attributes for Content+ if capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse)++ # To be removed once the change to span_attributes (from span events) is complete+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,+ },+ )+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,+ },+ )++ scope._span.set_status(Status(StatusCode.OK))++ # Metrics+ if not disable_metrics:+ record_completion_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, SemanticConvention.GEN_AI_SYSTEM_COHERE,+ scope._server_address, scope._server_port, request_model, scope._response_model, environment,+ application_name, scope._start_time, scope._end_time, scope._input_tokens, scope._output_tokens,+ cost, scope._tbt, scope._ttft)++def process_streaming_chat_response(scope, pricing_info, environment, application_name, metrics,+ capture_message_content=False, disable_metrics=False, version=""):+ """+ Process streaming chat request and generate Telemetry+ """++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=True)++def process_chat_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, start_time, span, capture_message_content=False,+ disable_metrics=False, version="1.0.0", **kwargs):+ """+ Process chat request and generate Telemetry+ """++ # Create scope object+ scope = type("GenericScope", (), {})()+ response_dict = response_as_dict(response)++ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ # Extract response content - handle both text and tool responses+ message = response_dict.get("message", {})+ content_list = message.get("content", [])+ if content_list and isinstance(content_list, list) and len(content_list) > 0:+ scope._llmresponse = content_list[0].get("text", "")+ else:+ scope._llmresponse = ""+ scope._response_id = response_dict.get("id")+ scope._response_model = request_model+ scope._input_tokens = response_dict.get("usage", {}).get("billed_units", {}).get("input_tokens", 0)+ scope._output_tokens = response_dict.get("usage", {}).get("billed_units", {}).get("output_tokens", 0)+ scope._timestamps = []+ scope._ttft, scope._tbt = scope._end_time - scope._start_time, 0+ scope._server_address, scope._server_port = server_address, server_port+ scope._kwargs = kwargs+ scope._finish_reason = response_dict.get("finish_reason", "")++ # Handle tool calls+ if scope._kwargs.get("tools"):+ scope._tools = response_dict.get("message", {}).get("tool_calls")+ # Handle tool plan if present+ scope._tool_plan = response_dict.get("message", {}).get("tool_plan", "")+ else:+ scope._tools = None+ scope._tool_plan = ""++ common_chat_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version, is_stream=False)++ return response++def common_embedding_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version):+ """+ Process embedding request and generate Telemetry+ """++ request_model = scope._kwargs.get("model", "embed-english-v3.0")+ inputs = scope._kwargs.get("texts", [])++ cost = get_embed_model_cost(request_model, pricing_info, scope._input_tokens)++ # Common Span Attributes+ common_span_attributes(scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING, SemanticConvention.GEN_AI_SYSTEM_COHERE,+ scope._server_address, scope._server_port, request_model, scope._response_model,+ environment, application_name, False, 0, scope._ttft, version)++ # Span Attributes for Request parameters+ scope._span.set_attribute(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS, scope._kwargs.get("embedding_types", ["float"]))++ # Span Attributes for Cost and Tokens+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, scope._input_tokens)+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)+ scope._span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, scope._response_type)++ # Span Attributes for Content+ if capture_message_content:+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, str(inputs))++ # To be removed once the change to span_attributes (from span events) is complete+ scope._span.add_event(+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes={+ SemanticConvention.GEN_AI_CONTENT_PROMPT: str(inputs),+ },+ )++ scope._span.set_status(Status(StatusCode.OK))++ # Metrics+ if not disable_metrics:+ record_embedding_metrics(metrics, SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING, SemanticConvention.GEN_AI_SYSTEM_COHERE,+ scope._server_address, scope._server_port, request_model, scope._response_model, environment,+ application_name, scope._start_time, scope._end_time, scope._input_tokens, cost)++def process_embedding_response(response, request_model, pricing_info, server_port, server_address,+ environment, application_name, metrics, start_time, span, capture_message_content=False,+ disable_metrics=False, version="1.0.0", **kwargs):+ """+ Process embedding request and generate Telemetry+ """++ # Create scope object+ scope = type("GenericScope", (), {})()+ response_dict = response_as_dict(response)++ scope._start_time = start_time+ scope._end_time = time.time()+ scope._span = span+ scope._input_tokens = response_dict.get("meta", {}).get("billed_units", {}).get("input_tokens", 0)+ scope._response_model = request_model+ scope._response_type = response_dict.get("response_type", "")+ scope._ttft = scope._end_time - scope._start_time+ scope._server_address, scope._server_port = server_address, server_port+ scope._kwargs = kwargs++ common_embedding_logic(scope, pricing_info, environment, application_name, metrics,+ capture_message_content, disable_metrics, version)++ return response