Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
Add chnages for Amazon Bedrock (#205)
patcher9b610f73ceb42
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.ymlindex 68aa98c4..3a8aeacd 100644--- a/.github/workflows/pypi-release.yml+++ b/.github/workflows/pypi-release.yml@@ -35,4 +35,4 @@ jobs:with:user: __token__password: ${{ secrets.PYPI_API_TOKEN }}- packages_dir: sdk/python/dist+ packages-dir: sdk/python/distdiff --git a/.github/workflows/test-pypi-release.yml b/.github/workflows/test-pypi-release.ymlindex e1ec5a56..190d15e0 100644--- a/.github/workflows/test-pypi-release.yml+++ b/.github/workflows/test-pypi-release.yml@@ -64,6 +64,6 @@ jobs:uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450with:user: __token__- repository_url: https://test.pypi.org/legacy/+ repository-url: https://test.pypi.org/legacy/password: ${{ secrets.TEST_PYPI_API_TOKEN }}- packages_dir: sdk/python/dist+ packages-dir: sdk/python/distdiff --git a/.gitignore b/.gitignoreindex 6597ade3..ac9be207 100644--- a/.gitignore+++ b/.gitignore@@ -4,4 +4,6 @@ test.pyyala.pyyala.ymlspeech.mp3-.venv\ No newline at end of file+.venv+path/+__pycache__\ No newline at end of filediff --git a/assets/pricing.json b/assets/pricing.jsonindex 9548dfe7..1d830df4 100644--- a/assets/pricing.json+++ b/assets/pricing.json@@ -19,7 +19,9 @@"embed-english-v2.0": 0.0001,"embed-english-light-v2.0": 0.0001,"embed-multilingual-v2.0": 0.0001,- "mistral-embed": 0.0001+ "mistral-embed": 0.0001,+ "amazon.titan-embed-text-v1": 0.0001,+ "amazon.titan-embed-text-v2": 0.00002},"images": {"dall-e-3": {@@ -52,6 +54,16 @@"512x512": 0.018,"256x256": 0.016}+ },+ "amazon.titan-image-generator-v1": {+ "standard": {+ "512x512": 0.008,+ "1024x1024": 0.01+ },+ "premium": {+ "512x512": 0.01,+ "1024x1024": 0.012+ }}},"audio": {@@ -59,54 +71,6 @@"tts-1-hd": 0.030},"chat": {- "ada": {- "promptPrice": 0.0004,- "completionPrice": 0.0004- },- "text-ada-001": {- "promptPrice": 0.0004,- "completionPrice": 0.0004- },- "babbage": {- "promptPrice": 0.0005,- "completionPrice": 0.0005- },- "babbage-002": {- "promptPrice": 0.0004,- "completionPrice": 0.0004- },- "text-babbage-001": {- "promptPrice": 0.0005,- "completionPrice": 0.0005- },- "curie": {- "promptPrice": 0.0020,- "completionPrice": 0.0020- },- "text-curie-001": {- "promptPrice": 0.0020,- "completionPrice": 0.0020- },- "davinci": {- "promptPrice": 0.0200,- "completionPrice": 0.0200- },- "davinci-002": {- "promptPrice": 0.0020,- "completionPrice": 0.0020- },- "text-davinci-001": {- "promptPrice": 0.0200,- "completionPrice": 0.0200- },- "text-davinci-002": {- "promptPrice": 0.0200,- "completionPrice": 0.0200- },- "text-davinci-003": {- "promptPrice": 0.0200,- "completionPrice": 0.0200- },"gpt-3.5-turbo": {"promptPrice": 0.0005,"completionPrice": 0.0015@@ -135,6 +99,10 @@"promptPrice": 0.03,"completionPrice": 0.06},+ "gpt-4-turbo": {+ "promptPrice": 0.01,+ "completionPrice": 0.03+ },"gpt-4-32k": {"promptPrice": 0.06,"completionPrice": 0.12@@ -214,6 +182,74 @@"mistral-large-latest": {"promptPrice": 0.008,"completionPrice": 0.024+ },+ "amazon.titan-text-express-v1": {+ "promptPrice": 0.0008,+ "completionPrice": 0.0016+ },+ "amazon.titan-text-lite-v1": {+ "promptPrice": 0.0003,+ "completionPrice": 0.0004+ },+ "mistral.mistral-7b-instruct-v0:2": {+ "promptPrice": 0.00015,+ "completionPrice": 0.0002+ },+ "mistral.mixtral-8x7b-instruct-v0:1": {+ "promptPrice": 0.00045,+ "completionPrice": 0.0007+ },+ "mistral.mistral-large-2402-v1:0": {+ "promptPrice": 0.008,+ "completionPrice": 0.024+ },+ "anthropic.claude-3-haiku-20240307-v1:0": {+ "promptPrice": 0.00025,+ "completionPrice": 0.00125+ },+ "anthropic.claude-v2": {+ "promptPrice": 0.008,+ "completionPrice": 0.024+ },+ "anthropic.claude-3-sonnet-20240229-v1:0": {+ "promptPrice": 0.003,+ "completionPrice": 0.015+ },+ "anthropic.claude-3-opus-20240229-v1:0": {+ "promptPrice": 0.015,+ "completionPrice": 0.075+ },+ "meta.llama3-8b-instruct-v1:0": {+ "promptPrice": 0.0004,+ "completionPrice": 0.0006+ },+ "meta.llama3-70b-instruct-v1:0": {+ "promptPrice": 0.00265,+ "completionPrice": 0.0035+ },+ "meta.llama2-13b-chat-v1": {+ "promptPrice": 0.00075,+ "completionPrice": 0.001+ },+ "meta.llama2-70b-chat-v1": {+ "promptPrice": 0.00195,+ "completionPrice": 0.00256+ },+ "cohere.command-text-v14": {+ "promptPrice": 0.0015,+ "completionPrice": 0.0020+ },+ "cohere.command-light-text-v14": {+ "promptPrice": 0.0003,+ "completionPrice": 0.0006+ },+ "ai21.j2-mid-v1": {+ "promptPrice": 0.0125,+ "completionPrice": 0.0125+ },+ "ai21.j2-ultra-v1": {+ "promptPrice": 0.0188,+ "completionPrice": 0.0188}}}diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex f8625d08..ebfa6654 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -12,6 +12,8 @@ keywords = ["OpenTelemetry", "otel", "otlp","llm", "tracing", "openai", "anthroppython = "^3.7.1"requests = "^2.26.0"tiktoken = "^0.1.1"+boto3 = "^1.34.0"+botocore = "^1.34.0"opentelemetry-api = "^1.24.0"opentelemetry-sdk = "^1.24.0"opentelemetry-exporter-otlp = "^1.24.0"diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex bff6cd52..09f6c714 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -17,6 +17,7 @@ from openlit.instrumentation.openai import OpenAIInstrumentorfrom openlit.instrumentation.anthropic import AnthropicInstrumentorfrom openlit.instrumentation.cohere import CohereInstrumentorfrom openlit.instrumentation.mistral import MistralInstrumentor+from openlit.instrumentation.bedrock import BedrockInstrumentorfrom openlit.instrumentation.langchain import LangChainInstrumentorfrom openlit.instrumentation.chroma import ChromaInstrumentorfrom openlit.instrumentation.pinecone import PineconeInstrumentor@@ -145,7 +146,8 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"openai": "openai","anthropic": "anthropic","cohere": "cohere",- "mistral": "mistralai",+ "mistral": "mistralai",+ "bedrock": "boto3","langchain": "langchain","chroma": "chromadb","pinecone": "pincone",@@ -192,6 +194,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"anthropic": AnthropicInstrumentor(),"cohere": CohereInstrumentor(),"mistral": MistralInstrumentor(),+ "bedrock": BedrockInstrumentor(),"langchain": LangChainInstrumentor(),"chroma": ChromaInstrumentor(),"pinecone": PineconeInstrumentor(),diff --git a/sdk/python/src/openlit/instrumentation/bedrock/__init__.py b/sdk/python/src/openlit/instrumentation/bedrock/__init__.pynew file mode 100644index 00000000..e330fd30--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/bedrock/__init__.py@@ -0,0 +1,41 @@+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code+"""Initializer of Auto Instrumentation of AWS Bedrock Functions"""++from typing import Collection+import importlib.metadata+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor+from wrapt import wrap_function_wrapper++from openlit.instrumentation.bedrock.bedrock import chat++_instruments = ("boto3 >= 1.34.93",)++class BedrockInstrumentor(BaseInstrumentor):+ """+ An instrumentor for AWS Bedrock's client library.+ """++ def instrumentation_dependencies(self) -> Collection[str]:+ return _instruments++ def _instrument(self, **kwargs):+ application_name = kwargs.get("application_name", "default_application")+ environment = kwargs.get("environment", "default_environment")+ tracer = kwargs.get("tracer")+ metrics = kwargs.get("metrics_dict")+ pricing_info = kwargs.get("pricing_info", {})+ trace_content = kwargs.get("trace_content", False)+ disable_metrics = kwargs.get("disable_metrics")+ version = importlib.metadata.version("boto3")++ #sync+ wrap_function_wrapper(+ "botocore.client",+ "ClientCreator.create_client",+ chat("bedrock.invoke_model", version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),+ )++ def _uninstrument(self, **kwargs):+ # Proper uninstrumentation logic to revert patched methods+ passdiff --git a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.pynew file mode 100644index 00000000..ee2005e3--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py@@ -0,0 +1,436 @@+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-branches+"""+Module for monitoring Amazon Bedrock API calls.+"""++import logging+import json+from botocore.response import StreamingBody+from botocore.exceptions import ReadTimeoutError, ResponseStreamingError+from urllib3.exceptions import ProtocolError as URLLib3ProtocolError+from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError+from opentelemetry.trace import SpanKind, Status, StatusCode+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, get_image_model_cost+from openlit.__helpers import handle_exception, general_tokens+from openlit.semcov import SemanticConvetion++# Initialize logger for logging potential issues and operations+logger = logging.getLogger(__name__)++class CustomStreamWrapper(StreamingBody):+ """Handle streaming responses with the ability to read multiple times."""++ def __init__(self, stream_source, length):+ super().__init__(stream_source, length)+ self._stream_data = None+ self._read_position = 0++ def read(self, amt=None):+ if self._stream_data is None:+ try:+ self._stream_data = self._raw_stream.read()+ except URLLib3ReadTimeoutError as error:+ raise ReadTimeoutError(endpoint_url=error.url, error=error) from error+ except URLLib3ProtocolError as error:+ raise ResponseStreamingError(error=error) from error++ self._amount_read += len(self._stream_data)+ if amt is None or (not self._stream_data and amt > 0):+ self._verify_content_length()++ if amt is None:+ data_chunk = self._stream_data[self._read_position:]+ else:+ data_start = self._read_position+ self._read_position += amt+ data_chunk = self._stream_data[data_start:self._read_position]++ return data_chunk+++def chat(gen_ai_endpoint, version, environment, application_name, tracer,+ pricing_info, trace_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for messages to collect metrics.++ Args:+ gen_ai_endpoint: Endpoint identifier for logging and tracing.+ version: The monitoring package version.+ environment: Deployment environment (e.g. production, staging).+ application_name: Name of the application using the Bedrock API.+ tracer: OpenTelemetry tracer for creating spans.+ pricing_info: Information for calculating Bedrock usage cost.+ trace_content: Whether to trace the actual content.+ metrics: Metrics collector.+ disable_metrics: Flag to toggle metrics collection.+ Returns:+ A function that wraps the chat method to add telemetry.+ """++ def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps an API call to add telemetry.++ Args:+ wrapped: Original method.+ instance: Instance of the class.+ args: Positional arguments of the 'messages' method.+ kwargs: Keyword arguments of the 'messages' method.+ Returns:+ Response from the original method.+ """+ def handle_image(span, model, request_body, response_body):+ cost = 0+ if "amazon" in model:+ # pylint: disable=line-too-long+ size = str(request_body.get("imageGenerationConfig", {}).get("width", 1024)) + "x" + str(request_body.get("imageGenerationConfig", {}).get("height", 1024))+ quality = request_body.get("imageGenerationConfig", {}).get("quality", "standard")+ n = request_body.get("imageGenerationConfig", {}).get("numberOfImages", 1)++ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,+ size)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,+ quality)+ # Calculate cost of the operation+ cost = n * get_image_model_cost(model,+ pricing_info, size, quality)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body.get("textToImageParams")["text"])++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_IMAGE,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_cost"].record(cost, attributes)++ def handle_embed(span, model, request_body, response_body):+ prompt_tokens, cost = 0, 0+ if "amazon" in model:+ prompt_tokens = response_body["inputTextTokenCount"]+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens)+ # Calculate cost of the operation+ cost = get_embed_model_cost(model,+ pricing_info, prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["inputText"])++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(+ prompt_tokens, attributes+ )+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ def handle_chat(span, model, request_body, response_body):+ prompt_tokens, completion_tokens, cost = 0, 0, 0++ if "amazon" in model:+ prompt_tokens = response_body["inputTextTokenCount"]+ completion_tokens = response_body["results"][0]["tokenCount"]+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ completion_tokens ++ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["results"][0]["completionReason"])++ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["inputText"])+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["results"][0]["outputText"])++ elif "mistral" in model:+ prompt_tokens = general_tokens(request_body["prompt"])+ completion_tokens = general_tokens(response_body["outputs"][0]["text"])+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens + completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["outputs"][0]["stop_reason"])+ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["prompt"])+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["outputs"][0]["text"])++ elif "anthropic" in model:+ prompt_tokens = response_body["usage"]["input_tokens"]+ completion_tokens = response_body["usage"]["output_tokens"]+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ completion_tokens ++ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["stop_reason"])++ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ # Format 'messages' into a single string+ message_prompt = request_body["messages"]+ formatted_messages = []+ for message in message_prompt:+ role = message["role"]+ content = message["content"]++ if isinstance(content, list):+ content_str = ", ".join(+ # pylint: disable=line-too-long+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'+ if "type" in item else f'text: {item["text"]}'+ for item in content+ )+ formatted_messages.append(f"{role}: {content_str}")+ else:+ formatted_messages.append(f"{role}: {content}")+ prompt = "\n".join(formatted_messages)+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ prompt)++ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["content"][0]["text"])+ elif "meta" in model:+ prompt_tokens = response_body["prompt_token_count"]+ completion_tokens = response_body["generation_token_count"]+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ completion_tokens ++ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["stop_reason"])++ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["prompt"])+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["generation"])++ elif "cohere" in model and "command-r" not in model:+ prompt_tokens = general_tokens(request_body["prompt"])+ completion_tokens = general_tokens(response_body["generations"][0]["text"])+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens + completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["generations"][0]["finish_reason"])+ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["prompt"])+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["generations"][0]["text"])+ elif "ai21" in model:+ prompt_tokens = general_tokens(request_body["prompt"])+ completion_tokens = general_tokens(response_body["completions"][0]["data"]["text"])+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens + completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,+ response_body["completions"][0]["finishReason"]["reason"])+ # Calculate cost of the operation+ cost = get_chat_model_cost(model,+ pricing_info, prompt_tokens,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ request_body["prompt"])+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response_body["completions"][0]["data"]["text"])++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(+ prompt_tokens + completion_tokens, attributes+ )+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ def add_instrumentation(original_method, *method_args, **method_kwargs):+ """+ Adds instrumentation to the invoke model call.++ Args:+ original_method: The original invoke model method.+ *method_args: Positional arguments for the method.+ **method_kwargs: Keyword arguments for the method.+ Returns:+ The modified response with telemetry.+ """+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:+ response = original_method(*method_args, **method_kwargs)++ try:+ # Modify the response body to be reusable+ response["body"] = CustomStreamWrapper(+ response["body"]._raw_stream, response["body"]._content_length+ )+ request_body = json.loads(method_kwargs.get("body"))+ response_body = json.loads(response.get("body").read())++ model = method_kwargs.get("modelId", "amazon.titan-text-express-v1")+ if ("stability" in model or "image" in model) and "embed-image" not in model:+ generation = "image"+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_IMAGE)+ elif "embed" in model and "embed-image" not in model:+ generation = "embeddings"+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)+ else:+ generation = "chat"+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_CHAT)++ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK)+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ model)+ if generation == "chat":+ handle_chat(span, model, request_body, response_body)+ elif generation == "embeddings":+ handle_embed(span, model, request_body, response_body)+ elif generation == "image":+ handle_image(span, model, request_body, response_body)++ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return response++ # Get the original client instance from the wrapper+ client = wrapped(*args, **kwargs)++ # Replace the original method with the instrumented one+ original_invoke_model = client.invoke_model+ client.invoke_model = lambda *args, **kwargs: add_instrumentation(original_invoke_model,+ *args, **kwargs)++ return client++ return wrapperdiff --git a/sdk/python/src/openlit/otel/metrics.py b/sdk/python/src/openlit/otel/metrics.pyindex 7dca7864..7af98a7e 100644--- a/sdk/python/src/openlit/otel/metrics.py+++ b/sdk/python/src/openlit/otel/metrics.py@@ -72,7 +72,7 @@ def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_headermetrics_dict = {"genai_requests": meter.create_counter(name=SemanticConvetion.GEN_AI_REQUESTS,- description="Number of requests to OpenAI",+ description="Number of requests to GenAI",unit="1",),"genai_prompt_tokens": meter.create_counter(@@ -92,7 +92,7 @@ def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_header),"genai_cost": meter.create_histogram(name=SemanticConvetion.GEN_AI_USAGE_COST,- description="The distribution of OpenAI request costs.",+ description="The distribution of GenAI request costs.",unit="USD",),"db_requests": meter.create_counter(diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 03556a08..2f12b23e 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -85,6 +85,7 @@ class SemanticConvetion:GEN_AI_SYSTEM_ANTHROPIC = "anthropic"GEN_AI_SYSTEM_COHERE = "cohere"GEN_AI_SYSTEM_MISTRAL = "mistral"+ GEN_AI_SYSTEM_BEDROCK = "bedrock"GEN_AI_SYSTEM_LANGCHAIN = "langchain"# Vector DBdiff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txtindex bb428aad..f8fc8983 100644--- a/sdk/python/tests/requirements.txt+++ b/sdk/python/tests/requirements.txt@@ -10,6 +10,8 @@ openai>=1.1.1anthropic>=0.21.0mistralai>=0.1.0transformers>=4.39.3+boto3>=1.34.93+botocore>=1.34.93# Vector DBschromadb>=0.4.0diff --git a/sdk/python/tests/test_anthropic.py.hold b/sdk/python/tests/test_anthropic.pysimilarity index 100%rename from sdk/python/tests/test_anthropic.py.holdrename to sdk/python/tests/test_anthropic.pydiff --git a/sdk/python/tests/test_cohere.py.hold b/sdk/python/tests/test_cohere.pysimilarity index 100%rename from sdk/python/tests/test_cohere.py.holdrename to sdk/python/tests/test_cohere.pydiff --git a/sdk/python/tests/test_mistral.py.hold b/sdk/python/tests/test_mistral.pysimilarity index 100%rename from sdk/python/tests/test_mistral.py.holdrename to sdk/python/tests/test_mistral.pydiff --git a/sdk/python/tests/test_openai.py.hold b/sdk/python/tests/test_openai.pysimilarity index 100%rename from sdk/python/tests/test_openai.py.holdrename to sdk/python/tests/test_openai.py