Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Add chnages for Amazon Bedrock (#205)

patcher9b610f73ceb42
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
index 68aa98c4..3a8aeacd 100644
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@@ -35,4 +35,4 @@ jobs:
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
- packages_dir: sdk/python/dist
+ packages-dir: sdk/python/dist
diff --git a/.github/workflows/test-pypi-release.yml b/.github/workflows/test-pypi-release.yml
index e1ec5a56..190d15e0 100644
--- a/.github/workflows/test-pypi-release.yml
+++ b/.github/workflows/test-pypi-release.yml
@@ -64,6 +64,6 @@ jobs:
uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450
with:
user: __token__
- repository_url: https://test.pypi.org/legacy/
+ repository-url: https://test.pypi.org/legacy/
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
- packages_dir: sdk/python/dist
+ packages-dir: sdk/python/dist
diff --git a/.gitignore b/.gitignore
index 6597ade3..ac9be207 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,6 @@ test.py
yala.py
yala.yml
speech.mp3
-.venv
\ No newline at end of file
+.venv
+path/
+__pycache__
\ No newline at end of file
diff --git a/assets/pricing.json b/assets/pricing.json
index 9548dfe7..1d830df4 100644
--- a/assets/pricing.json
+++ b/assets/pricing.json
@@ -19,7 +19,9 @@
"embed-english-v2.0": 0.0001,
"embed-english-light-v2.0": 0.0001,
"embed-multilingual-v2.0": 0.0001,
- "mistral-embed": 0.0001
+ "mistral-embed": 0.0001,
+ "amazon.titan-embed-text-v1": 0.0001,
+ "amazon.titan-embed-text-v2": 0.00002
},
"images": {
"dall-e-3": {
@@ -52,6 +54,16 @@
"512x512": 0.018,
"256x256": 0.016
}
+ },
+ "amazon.titan-image-generator-v1": {
+ "standard": {
+ "512x512": 0.008,
+ "1024x1024": 0.01
+ },
+ "premium": {
+ "512x512": 0.01,
+ "1024x1024": 0.012
+ }
}
},
"audio": {
@@ -59,54 +71,6 @@
"tts-1-hd": 0.030
},
"chat": {
- "ada": {
- "promptPrice": 0.0004,
- "completionPrice": 0.0004
- },
- "text-ada-001": {
- "promptPrice": 0.0004,
- "completionPrice": 0.0004
- },
- "babbage": {
- "promptPrice": 0.0005,
- "completionPrice": 0.0005
- },
- "babbage-002": {
- "promptPrice": 0.0004,
- "completionPrice": 0.0004
- },
- "text-babbage-001": {
- "promptPrice": 0.0005,
- "completionPrice": 0.0005
- },
- "curie": {
- "promptPrice": 0.0020,
- "completionPrice": 0.0020
- },
- "text-curie-001": {
- "promptPrice": 0.0020,
- "completionPrice": 0.0020
- },
- "davinci": {
- "promptPrice": 0.0200,
- "completionPrice": 0.0200
- },
- "davinci-002": {
- "promptPrice": 0.0020,
- "completionPrice": 0.0020
- },
- "text-davinci-001": {
- "promptPrice": 0.0200,
- "completionPrice": 0.0200
- },
- "text-davinci-002": {
- "promptPrice": 0.0200,
- "completionPrice": 0.0200
- },
- "text-davinci-003": {
- "promptPrice": 0.0200,
- "completionPrice": 0.0200
- },
"gpt-3.5-turbo": {
"promptPrice": 0.0005,
"completionPrice": 0.0015
@@ -135,6 +99,10 @@
"promptPrice": 0.03,
"completionPrice": 0.06
},
+ "gpt-4-turbo": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
"gpt-4-32k": {
"promptPrice": 0.06,
"completionPrice": 0.12
@@ -214,6 +182,74 @@
"mistral-large-latest": {
"promptPrice": 0.008,
"completionPrice": 0.024
+ },
+ "amazon.titan-text-express-v1": {
+ "promptPrice": 0.0008,
+ "completionPrice": 0.0016
+ },
+ "amazon.titan-text-lite-v1": {
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0004
+ },
+ "mistral.mistral-7b-instruct-v0:2": {
+ "promptPrice": 0.00015,
+ "completionPrice": 0.0002
+ },
+ "mistral.mixtral-8x7b-instruct-v0:1": {
+ "promptPrice": 0.00045,
+ "completionPrice": 0.0007
+ },
+ "mistral.mistral-large-2402-v1:0": {
+ "promptPrice": 0.008,
+ "completionPrice": 0.024
+ },
+ "anthropic.claude-3-haiku-20240307-v1:0": {
+ "promptPrice": 0.00025,
+ "completionPrice": 0.00125
+ },
+ "anthropic.claude-v2": {
+ "promptPrice": 0.008,
+ "completionPrice": 0.024
+ },
+ "anthropic.claude-3-sonnet-20240229-v1:0": {
+ "promptPrice": 0.003,
+ "completionPrice": 0.015
+ },
+ "anthropic.claude-3-opus-20240229-v1:0": {
+ "promptPrice": 0.015,
+ "completionPrice": 0.075
+ },
+ "meta.llama3-8b-instruct-v1:0": {
+ "promptPrice": 0.0004,
+ "completionPrice": 0.0006
+ },
+ "meta.llama3-70b-instruct-v1:0": {
+ "promptPrice": 0.00265,
+ "completionPrice": 0.0035
+ },
+ "meta.llama2-13b-chat-v1": {
+ "promptPrice": 0.00075,
+ "completionPrice": 0.001
+ },
+ "meta.llama2-70b-chat-v1": {
+ "promptPrice": 0.00195,
+ "completionPrice": 0.00256
+ },
+ "cohere.command-text-v14": {
+ "promptPrice": 0.0015,
+ "completionPrice": 0.0020
+ },
+ "cohere.command-light-text-v14": {
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0006
+ },
+ "ai21.j2-mid-v1": {
+ "promptPrice": 0.0125,
+ "completionPrice": 0.0125
+ },
+ "ai21.j2-ultra-v1": {
+ "promptPrice": 0.0188,
+ "completionPrice": 0.0188
}
}
}
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index f8625d08..ebfa6654 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -12,6 +12,8 @@ keywords = ["OpenTelemetry", "otel", "otlp","llm", "tracing", "openai", "anthrop
python = "^3.7.1"
requests = "^2.26.0"
tiktoken = "^0.1.1"
+boto3 = "^1.34.0"
+botocore = "^1.34.0"
opentelemetry-api = "^1.24.0"
opentelemetry-sdk = "^1.24.0"
opentelemetry-exporter-otlp = "^1.24.0"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index bff6cd52..09f6c714 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -17,6 +17,7 @@ from openlit.instrumentation.openai import OpenAIInstrumentor
from openlit.instrumentation.anthropic import AnthropicInstrumentor
from openlit.instrumentation.cohere import CohereInstrumentor
from openlit.instrumentation.mistral import MistralInstrumentor
+from openlit.instrumentation.bedrock import BedrockInstrumentor
from openlit.instrumentation.langchain import LangChainInstrumentor
from openlit.instrumentation.chroma import ChromaInstrumentor
from openlit.instrumentation.pinecone import PineconeInstrumentor
@@ -145,7 +146,8 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"openai": "openai",
"anthropic": "anthropic",
"cohere": "cohere",
- "mistral": "mistralai",
+ "mistral": "mistralai",
+ "bedrock": "boto3",
"langchain": "langchain",
"chroma": "chromadb",
"pinecone": "pincone",
@@ -192,6 +194,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"anthropic": AnthropicInstrumentor(),
"cohere": CohereInstrumentor(),
"mistral": MistralInstrumentor(),
+ "bedrock": BedrockInstrumentor(),
"langchain": LangChainInstrumentor(),
"chroma": ChromaInstrumentor(),
"pinecone": PineconeInstrumentor(),
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/__init__.py b/sdk/python/src/openlit/instrumentation/bedrock/__init__.py
new file mode 100644
index 00000000..e330fd30
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/bedrock/__init__.py
@@ -0,0 +1,41 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of AWS Bedrock Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.bedrock.bedrock import chat
+
+_instruments = ("boto3 >= 1.34.93",)
+
+class BedrockInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for AWS Bedrock's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default_application")
+ environment = kwargs.get("environment", "default_environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("boto3")
+
+ #sync
+ wrap_function_wrapper(
+ "botocore.client",
+ "ClientCreator.create_client",
+ chat("bedrock.invoke_model", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py
new file mode 100644
index 00000000..ee2005e3
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py
@@ -0,0 +1,436 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, protected-access, too-many-branches
+"""
+Module for monitoring Amazon Bedrock API calls.
+"""
+
+import logging
+import json
+from botocore.response import StreamingBody
+from botocore.exceptions import ReadTimeoutError, ResponseStreamingError
+from urllib3.exceptions import ProtocolError as URLLib3ProtocolError
+from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, get_image_model_cost
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+class CustomStreamWrapper(StreamingBody):
+ """Handle streaming responses with the ability to read multiple times."""
+
+ def __init__(self, stream_source, length):
+ super().__init__(stream_source, length)
+ self._stream_data = None
+ self._read_position = 0
+
+ def read(self, amt=None):
+ if self._stream_data is None:
+ try:
+ self._stream_data = self._raw_stream.read()
+ except URLLib3ReadTimeoutError as error:
+ raise ReadTimeoutError(endpoint_url=error.url, error=error) from error
+ except URLLib3ProtocolError as error:
+ raise ResponseStreamingError(error=error) from error
+
+ self._amount_read += len(self._stream_data)
+ if amt is None or (not self._stream_data and amt > 0):
+ self._verify_content_length()
+
+ if amt is None:
+ data_chunk = self._stream_data[self._read_position:]
+ else:
+ data_start = self._read_position
+ self._read_position += amt
+ data_chunk = self._stream_data[data_start:self._read_position]
+
+ return data_chunk
+
+
+def chat(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: The monitoring package version.
+ environment: Deployment environment (e.g. production, staging).
+ application_name: Name of the application using the Bedrock API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information for calculating Bedrock usage cost.
+ trace_content: Whether to trace the actual content.
+ metrics: Metrics collector.
+ disable_metrics: Flag to toggle metrics collection.
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps an API call to add telemetry.
+
+ Args:
+ wrapped: Original method.
+ instance: Instance of the class.
+ args: Positional arguments of the 'messages' method.
+ kwargs: Keyword arguments of the 'messages' method.
+ Returns:
+ Response from the original method.
+ """
+ def handle_image(span, model, request_body, response_body):
+ cost = 0
+ if "amazon" in model:
+ # pylint: disable=line-too-long
+ size = str(request_body.get("imageGenerationConfig", {}).get("width", 1024)) + "x" + str(request_body.get("imageGenerationConfig", {}).get("height", 1024))
+ quality = request_body.get("imageGenerationConfig", {}).get("quality", "standard")
+ n = request_body.get("imageGenerationConfig", {}).get("numberOfImages", 1)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ size)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ quality)
+ # Calculate cost of the operation
+ cost = n * get_image_model_cost(model,
+ pricing_info, size, quality)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body.get("textToImageParams")["text"])
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ def handle_embed(span, model, request_body, response_body):
+ prompt_tokens, cost = 0, 0
+ if "amazon" in model:
+ prompt_tokens = response_body["inputTextTokenCount"]
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(model,
+ pricing_info, prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["inputText"])
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens, attributes
+ )
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ def handle_chat(span, model, request_body, response_body):
+ prompt_tokens, completion_tokens, cost = 0, 0, 0
+
+ if "amazon" in model:
+ prompt_tokens = response_body["inputTextTokenCount"]
+ completion_tokens = response_body["results"][0]["tokenCount"]
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ completion_tokens +
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["results"][0]["completionReason"])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["inputText"])
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["results"][0]["outputText"])
+
+ elif "mistral" in model:
+ prompt_tokens = general_tokens(request_body["prompt"])
+ completion_tokens = general_tokens(response_body["outputs"][0]["text"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["outputs"][0]["stop_reason"])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["prompt"])
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["outputs"][0]["text"])
+
+ elif "anthropic" in model:
+ prompt_tokens = response_body["usage"]["input_tokens"]
+ completion_tokens = response_body["usage"]["output_tokens"]
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ completion_tokens +
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["stop_reason"])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ # Format 'messages' into a single string
+ message_prompt = request_body["messages"]
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["content"][0]["text"])
+ elif "meta" in model:
+ prompt_tokens = response_body["prompt_token_count"]
+ completion_tokens = response_body["generation_token_count"]
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ completion_tokens +
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["stop_reason"])
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["prompt"])
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["generation"])
+
+ elif "cohere" in model and "command-r" not in model:
+ prompt_tokens = general_tokens(request_body["prompt"])
+ completion_tokens = general_tokens(response_body["generations"][0]["text"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["generations"][0]["finish_reason"])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["prompt"])
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["generations"][0]["text"])
+ elif "ai21" in model:
+ prompt_tokens = general_tokens(request_body["prompt"])
+ completion_tokens = general_tokens(response_body["completions"][0]["data"]["text"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response_body["completions"][0]["finishReason"]["reason"])
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model,
+ pricing_info, prompt_tokens,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ request_body["prompt"])
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response_body["completions"][0]["data"]["text"])
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ def add_instrumentation(original_method, *method_args, **method_kwargs):
+ """
+ Adds instrumentation to the invoke model call.
+
+ Args:
+ original_method: The original invoke model method.
+ *method_args: Positional arguments for the method.
+ **method_kwargs: Keyword arguments for the method.
+ Returns:
+ The modified response with telemetry.
+ """
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = original_method(*method_args, **method_kwargs)
+
+ try:
+ # Modify the response body to be reusable
+ response["body"] = CustomStreamWrapper(
+ response["body"]._raw_stream, response["body"]._content_length
+ )
+ request_body = json.loads(method_kwargs.get("body"))
+ response_body = json.loads(response.get("body").read())
+
+ model = method_kwargs.get("modelId", "amazon.titan-text-express-v1")
+ if ("stability" in model or "image" in model) and "embed-image" not in model:
+ generation = "image"
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ elif "embed" in model and "embed-image" not in model:
+ generation = "embeddings"
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ else:
+ generation = "chat"
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_BEDROCK)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ if generation == "chat":
+ handle_chat(span, model, request_body, response_body)
+ elif generation == "embeddings":
+ handle_embed(span, model, request_body, response_body)
+ elif generation == "image":
+ handle_image(span, model, request_body, response_body)
+
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ # Get the original client instance from the wrapper
+ client = wrapped(*args, **kwargs)
+
+ # Replace the original method with the instrumented one
+ original_invoke_model = client.invoke_model
+ client.invoke_model = lambda *args, **kwargs: add_instrumentation(original_invoke_model,
+ *args, **kwargs)
+
+ return client
+
+ return wrapper
diff --git a/sdk/python/src/openlit/otel/metrics.py b/sdk/python/src/openlit/otel/metrics.py
index 7dca7864..7af98a7e 100644
--- a/sdk/python/src/openlit/otel/metrics.py
+++ b/sdk/python/src/openlit/otel/metrics.py
@@ -72,7 +72,7 @@ def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_header
metrics_dict = {
"genai_requests": meter.create_counter(
name=SemanticConvetion.GEN_AI_REQUESTS,
- description="Number of requests to OpenAI",
+ description="Number of requests to GenAI",
unit="1",
),
"genai_prompt_tokens": meter.create_counter(
@@ -92,7 +92,7 @@ def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_header
),
"genai_cost": meter.create_histogram(
name=SemanticConvetion.GEN_AI_USAGE_COST,
- description="The distribution of OpenAI request costs.",
+ description="The distribution of GenAI request costs.",
unit="USD",
),
"db_requests": meter.create_counter(
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 03556a08..2f12b23e 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -85,6 +85,7 @@ class SemanticConvetion:
GEN_AI_SYSTEM_ANTHROPIC = "anthropic"
GEN_AI_SYSTEM_COHERE = "cohere"
GEN_AI_SYSTEM_MISTRAL = "mistral"
+ GEN_AI_SYSTEM_BEDROCK = "bedrock"
GEN_AI_SYSTEM_LANGCHAIN = "langchain"
# Vector DB
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index bb428aad..f8fc8983 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -10,6 +10,8 @@ openai>=1.1.1
anthropic>=0.21.0
mistralai>=0.1.0
transformers>=4.39.3
+boto3>=1.34.93
+botocore>=1.34.93
# Vector DBs
chromadb>=0.4.0
diff --git a/sdk/python/tests/test_anthropic.py.hold b/sdk/python/tests/test_anthropic.py
similarity index 100%
rename from sdk/python/tests/test_anthropic.py.hold
rename to sdk/python/tests/test_anthropic.py
diff --git a/sdk/python/tests/test_cohere.py.hold b/sdk/python/tests/test_cohere.py
similarity index 100%
rename from sdk/python/tests/test_cohere.py.hold
rename to sdk/python/tests/test_cohere.py
diff --git a/sdk/python/tests/test_mistral.py.hold b/sdk/python/tests/test_mistral.py
similarity index 100%
rename from sdk/python/tests/test_mistral.py.hold
rename to sdk/python/tests/test_mistral.py
diff --git a/sdk/python/tests/test_openai.py.hold b/sdk/python/tests/test_openai.py
similarity index 100%
rename from sdk/python/tests/test_openai.py.hold
rename to sdk/python/tests/test_openai.py