Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenTelemetry Instrumentation for Monitoring Together AI (#550)

Patcherc00969c796ed
diff --git a/assets/pricing.json b/assets/pricing.json
index ea358f4e..7b79cddb 100644
--- a/assets/pricing.json
+++ b/assets/pricing.json
@@ -59,6 +59,47 @@
"256x256": 0.016
}
},
+ "black-forest-labs/FLUX.1-dev": {
+ "standard": {
+ "1000000": 0.025
+ }
+ },
+ "black-forest-labs/FLUX.1-canny": {
+ "standard": {
+ "1000000": 0.025
+ }
+ },
+ "black-forest-labs/FLUX.1-depth": {
+ "standard": {
+ "1000000": 0.025
+ }
+ },
+ "black-forest-labs/FLUX.1-redux": {
+ "standard": {
+ "1000000": 0.025
+ }
+ },
+ "black-forest-labs/FLUX.1-schnell": {
+ "standard": {
+ "1000000": 0.0027
+ }
+ },
+ "black-forest-labs/FLUX.1-pro": {
+ "standard": {
+ "1000000": 0.04
+ }
+ },
+ "black-forest-labs/FLUX.1.1-pro": {
+ "standard": {
+ "1000000": 0.05
+ }
+ },
+ "stabilityai/stable-diffusion-xl-base-1.0": {
+ "standard": {
+ "512X512": 0.001,
+ "1024x1024": 0.01
+ }
+ },
"amazon.titan-image-generator-v1": {
"standard": {
"512x512": 0.008,
@@ -431,6 +472,58 @@
"jamba-1.5-large": {
"promptPrice": 0.002,
"completionPrice": 0.008
+ },
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo": {
+ "promptPrice": 0.00088,
+ "completionPrice": 0.00088
+ },
+ "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo-vLLM-json":{
+ "promptPrice": 0.00088,
+ "completionPrice": 0.00088
+ },
+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Lite-Pro-lora":{
+ "promptPrice": 0.00088,
+ "completionPrice": 0.00088
+ },
+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo":{
+ "promptPrice": 0.00088,
+ "completionPrice": 0.00088
+ },
+ "Qwen/QwQ-32B-Preview":{
+ "promptPrice": 0.0012,
+ "completionPrice": 0.0012
+ },
+ "codellama/CodeLlama-34b-Instruct-hf":{
+ "promptPrice": 0.0008,
+ "completionPrice":0.0008
+ },
+ "databricks/dbrx-instruct":{
+ "promptPrice": 0.0012,
+ "completionPrice": 0.0012
+ },
+ "deepseek-ai/deepseek-llm-67b-chat":{
+ "promptPrice": 0.0009,
+ "completionPrice": 0.0009
+ },
+ "google/gemma-2b-it":{
+ "promptPrice": 0.0001,
+ "completionPrice": 0.0001
+ },
+ "google/gemma-2-27b-it":{
+ "promptPrice": 0.0008,
+ "completionPrice": 0.0008
+ },
+ "google/gemma-2-9b-it":{
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0003
+ },
+ "Gryphe/MythoMax-L2-13b-Lite":{
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0003
+ },
+ "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF":{
+ "promptPrice": 0.0088,
+ "completionPrice": 0.0088
}
}
}
\ No newline at end of file
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 70908385..4da72bb3 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.33.0"
+version = "1.33.1"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/instrumentation/together/async_together.py b/sdk/python/src/openlit/instrumentation/together/async_together.py
index 0648b765..6c083f53 100644
--- a/sdk/python/src/openlit/instrumentation/together/async_together.py
+++ b/sdk/python/src/openlit/instrumentation/together/async_together.py
@@ -1,4 +1,4 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes
"""
Module for monitoring Together calls.
"""
@@ -8,7 +8,6 @@ from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
from openlit.__helpers import (
get_chat_model_cost,
- general_tokens,
get_image_model_cost,
handle_exception,
response_as_dict,
@@ -58,6 +57,9 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
# Placeholder for aggregating streaming response
self._llmresponse = ""
self._response_id = ""
+ self._prompt_tokens = 0
+ self._completion_tokens = 0
+ self._total_tokens = 0
self._args = args
self._kwargs = kwargs
@@ -116,7 +118,10 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
prompt = "\n".join(formatted_messages)
# Calculate cost of the operation
- cost = get_chat_model_cost(self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),
+ cost = get_chat_model_cost(self._kwargs.get(
+ "model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ ),
pricing_info, self._prompt_tokens,
self._completion_tokens)
@@ -135,7 +140,10 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
application_name)
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"))
+ self._kwargs.get(
+ "model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ ))
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
self._kwargs.get("user", ""))
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
@@ -189,15 +197,20 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
SemanticConvetion.GEN_AI_TYPE:
SemanticConvetion.GEN_AI_TYPE_CHAT,
SemanticConvetion.GEN_AI_REQUEST_MODEL:
- self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo")
+ self._kwargs.get("model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo")
}
metrics["genai_requests"].add(1, attributes)
metrics["genai_total_tokens"].add(
self._total_tokens, attributes
)
- metrics["genai_completion_tokens"].add(self._completion_tokens, attributes)
- metrics["genai_prompt_tokens"].add(self._prompt_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ self._completion_tokens, attributes
+ )
+ metrics["genai_prompt_tokens"].add(
+ self._prompt_tokens, attributes
+ )
metrics["genai_cost"].record(cost, attributes)
except Exception as e:
@@ -278,7 +291,8 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
application_name)
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"))
+ kwargs.get("model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
kwargs.get("top_p", 1.0))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
@@ -306,8 +320,12 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
# Set span attributes when tools is not passed to the function call
if "tools" not in kwargs:
# Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),
- pricing_info, response_dict.get('usage', {}).get('prompt_tokens', None),
+ cost = get_chat_model_cost(kwargs.get(
+ "model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ ),
+ pricing_info,
+ response_dict.get('usage', {}).get('prompt_tokens', None),
response_dict.get('usage', {}).get('completion_tokens', None))
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
@@ -348,9 +366,14 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
# Set span attributes when tools is passed to the function call
elif "tools" in kwargs:
# Calculate cost of the operation
- cost = get_chat_model_cost(kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),
- pricing_info, response_dict.get('usage').get('prompt_tokens'),
+ cost = get_chat_model_cost(kwargs.get(
+ "model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ ),
+ pricing_info,
+ response_dict.get('usage').get('prompt_tokens'),
response_dict.get('usage').get('completion_tokens'))
+
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
@@ -385,9 +408,12 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,
}
metrics["genai_requests"].add(1, attributes)
- metrics["genai_total_tokens"].add(response_dict.get('usage').get('total_tokens'), attributes)
- metrics["genai_completion_tokens"].add(response_dict.get('usage').get('completion_tokens'), attributes)
- metrics["genai_prompt_tokens"].add(response_dict.get('usage').get('prompt_tokens'), attributes)
+ metrics["genai_total_tokens"].add(
+ response_dict.get('usage').get('total_tokens'), attributes)
+ metrics["genai_completion_tokens"].add(
+ response_dict.get('usage').get('completion_tokens'), attributes)
+ metrics["genai_prompt_tokens"].add(
+ response_dict.get('usage').get('prompt_tokens'), attributes)
metrics["genai_cost"].record(cost, attributes)
# Return original response
@@ -449,10 +475,14 @@ def async_image_generate(gen_ai_endpoint, version, environment, application_name
image = "url"
# Calculate cost of the operation
- image_size = str(kwargs.get("height", 1024)) + "x" + str(kwargs.get("width", 1024))
- cost = get_image_model_cost(kwargs.get("model", "black-forest-labs/FLUX.1-dev"),
- pricing_info, image_size,
+ image_size = str(kwargs.get("width", 1024)) + "x" + str(kwargs.get("height", 1024))
+ cost_per_million = get_image_model_cost(kwargs.get(
+ "model", "black-forest-labs/FLUX.1-dev"
+ ),
+ pricing_info, "1000000",
kwargs.get("quality", "standard"))
+ pixels = kwargs.get("width", 1024) * kwargs.get("height", 1024)
+ cost = pixels / 1_000_000 * cost_per_million
for items in response.data:
# Set Span attributes
diff --git a/sdk/python/src/openlit/instrumentation/together/together.py b/sdk/python/src/openlit/instrumentation/together/together.py
index 6a64c8e3..0f9d4625 100644
--- a/sdk/python/src/openlit/instrumentation/together/together.py
+++ b/sdk/python/src/openlit/instrumentation/together/together.py
@@ -1,4 +1,4 @@
-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes
"""
Module for monitoring Together calls.
"""
@@ -8,7 +8,6 @@ from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
from openlit.__helpers import (
get_chat_model_cost,
- general_tokens,
get_image_model_cost,
handle_exception,
response_as_dict,
@@ -141,8 +140,10 @@ def completion(gen_ai_endpoint, version, environment, application_name,
self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
application_name)
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- self._kwargs.get("model",
- "meta-llama/Llama-3.3-70B-Instruct-Turbo"))
+ self._kwargs.get(
+ "model",
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+ ))
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
self._kwargs.get("user", ""))
self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
@@ -323,7 +324,7 @@ def completion(gen_ai_endpoint, version, environment, application_name,
"model",
"meta-llama/Llama-3.3-70B-Instruct-Turbo"
),
- pricing_info,
+ pricing_info,
response_dict.get('usage', {}).get('prompt_tokens', None),
response_dict.get('usage', {}).get('completion_tokens', None))
@@ -372,6 +373,7 @@ def completion(gen_ai_endpoint, version, environment, application_name,
pricing_info,
response_dict.get('usage').get('prompt_tokens'),
response_dict.get('usage').get('completion_tokens'))
+
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
attributes={
@@ -473,10 +475,14 @@ def image_generate(gen_ai_endpoint, version, environment, application_name,
image = "url"
# Calculate cost of the operation
- image_size = str(kwargs.get("height", 1024)) + "x" + str(kwargs.get("width", 1024))
- cost = get_image_model_cost(kwargs.get("model", "black-forest-labs/FLUX.1-dev"),
- pricing_info, image_size,
+ image_size = str(kwargs.get("width", 1024)) + "x" + str(kwargs.get("height", 1024))
+ cost_per_million = get_image_model_cost(kwargs.get(
+ "model", "black-forest-labs/FLUX.1-dev"
+ ),
+ pricing_info, "1000000",
kwargs.get("quality", "standard"))
+ pixels = kwargs.get("width", 1024) * kwargs.get("height", 1024)
+ cost = pixels / 1_000_000 * cost_per_million
for items in response.data:
# Set Span attributes
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index 79e0ace8..44cf5d2c 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -26,6 +26,7 @@ ai21
controlflow
assemblyai
firecrawl-py
+together
# Vector DBs
chromadb
diff --git a/sdk/python/tests/test_together.py b/sdk/python/tests/test_together.py
index e0bc9b08..04255fbc 100644
--- a/sdk/python/tests/test_together.py
+++ b/sdk/python/tests/test_together.py
@@ -13,7 +13,7 @@ prior to running these tests.
"""
import pytest
-from together import Together
+from together import Together, AsyncTogether
import openlit
# Initialize synchronous Together client
@@ -44,7 +44,7 @@ def test_sync_together_chat():
max_tokens=1,
stream=False,
)
- assert message.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'
+ assert response.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'
def test_sync_together_image():
"""
@@ -62,7 +62,7 @@ def test_sync_together_image():
n=1,
)
- assert message.model == 'black-forest-labs/FLUX.1-dev'
+ assert response.model == 'black-forest-labs/FLUX.1-dev'
@pytest.mark.asyncio
async def test_async_together_chat():
@@ -84,7 +84,7 @@ async def test_async_together_chat():
max_tokens=1,
stream=False,
)
- assert message.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'
+ assert response.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'
@pytest.mark.asyncio
async def test_async_together_image():
@@ -103,4 +103,4 @@ async def test_async_together_image():
n=1,
)
- assert message.model == 'black-forest-labs/FLUX.1-dev'
\ No newline at end of file
+ assert response.model == 'black-forest-labs/FLUX.1-dev'