Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Add OpenTelemetry Instrumentation for Monitoring Together AI (#550)
Patcherc00969c796ed
diff --git a/assets/pricing.json b/assets/pricing.jsonindex ea358f4e..7b79cddb 100644--- a/assets/pricing.json+++ b/assets/pricing.json@@ -59,6 +59,47 @@"256x256": 0.016}},+ "black-forest-labs/FLUX.1-dev": {+ "standard": {+ "1000000": 0.025+ }+ },+ "black-forest-labs/FLUX.1-canny": {+ "standard": {+ "1000000": 0.025+ }+ },+ "black-forest-labs/FLUX.1-depth": {+ "standard": {+ "1000000": 0.025+ }+ },+ "black-forest-labs/FLUX.1-redux": {+ "standard": {+ "1000000": 0.025+ }+ },+ "black-forest-labs/FLUX.1-schnell": {+ "standard": {+ "1000000": 0.0027+ }+ },+ "black-forest-labs/FLUX.1-pro": {+ "standard": {+ "1000000": 0.04+ }+ },+ "black-forest-labs/FLUX.1.1-pro": {+ "standard": {+ "1000000": 0.05+ }+ },+ "stabilityai/stable-diffusion-xl-base-1.0": {+ "standard": {+ "512X512": 0.001,+ "1024x1024": 0.01+ }+ },"amazon.titan-image-generator-v1": {"standard": {"512x512": 0.008,@@ -431,6 +472,58 @@"jamba-1.5-large": {"promptPrice": 0.002,"completionPrice": 0.008+ },+ "meta-llama/Llama-3.3-70B-Instruct-Turbo": {+ "promptPrice": 0.00088,+ "completionPrice": 0.00088+ },+ "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo-vLLM-json":{+ "promptPrice": 0.00088,+ "completionPrice": 0.00088+ },+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Lite-Pro-lora":{+ "promptPrice": 0.00088,+ "completionPrice": 0.00088+ },+ "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo":{+ "promptPrice": 0.00088,+ "completionPrice": 0.00088+ },+ "Qwen/QwQ-32B-Preview":{+ "promptPrice": 0.0012,+ "completionPrice": 0.0012+ },+ "codellama/CodeLlama-34b-Instruct-hf":{+ "promptPrice": 0.0008,+ "completionPrice":0.0008+ },+ "databricks/dbrx-instruct":{+ "promptPrice": 0.0012,+ "completionPrice": 0.0012+ },+ "deepseek-ai/deepseek-llm-67b-chat":{+ "promptPrice": 0.0009,+ "completionPrice": 0.0009+ },+ "google/gemma-2b-it":{+ "promptPrice": 0.0001,+ "completionPrice": 0.0001+ },+ "google/gemma-2-27b-it":{+ "promptPrice": 0.0008,+ "completionPrice": 0.0008+ },+ "google/gemma-2-9b-it":{+ "promptPrice": 0.0003,+ "completionPrice": 0.0003+ },+ "Gryphe/MythoMax-L2-13b-Lite":{+ "promptPrice": 0.0003,+ "completionPrice": 0.0003+ },+ "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF":{+ "promptPrice": 0.0088,+ "completionPrice": 0.0088}}}\ No newline at end of filediff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 70908385..4da72bb3 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.33.0"+version = "1.33.1"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/instrumentation/together/async_together.py b/sdk/python/src/openlit/instrumentation/together/async_together.pyindex 0648b765..6c083f53 100644--- a/sdk/python/src/openlit/instrumentation/together/async_together.py+++ b/sdk/python/src/openlit/instrumentation/together/async_together.py@@ -1,4 +1,4 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes"""Module for monitoring Together calls."""@@ -8,7 +8,6 @@ from opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import TELEMETRY_SDK_NAMEfrom openlit.__helpers import (get_chat_model_cost,- general_tokens,get_image_model_cost,handle_exception,response_as_dict,@@ -58,6 +57,9 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,# Placeholder for aggregating streaming responseself._llmresponse = ""self._response_id = ""+ self._prompt_tokens = 0+ self._completion_tokens = 0+ self._total_tokens = 0self._args = argsself._kwargs = kwargs@@ -116,7 +118,10 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,prompt = "\n".join(formatted_messages)# Calculate cost of the operation- cost = get_chat_model_cost(self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),+ cost = get_chat_model_cost(self._kwargs.get(+ "model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"+ ),pricing_info, self._prompt_tokens,self._completion_tokens)@@ -135,7 +140,10 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,application_name)self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"))+ self._kwargs.get(+ "model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"+ ))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,self._kwargs.get("user", ""))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,@@ -189,15 +197,20 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,SemanticConvetion.GEN_AI_TYPE:SemanticConvetion.GEN_AI_TYPE_CHAT,SemanticConvetion.GEN_AI_REQUEST_MODEL:- self._kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo")+ self._kwargs.get("model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo")}metrics["genai_requests"].add(1, attributes)metrics["genai_total_tokens"].add(self._total_tokens, attributes)- metrics["genai_completion_tokens"].add(self._completion_tokens, attributes)- metrics["genai_prompt_tokens"].add(self._prompt_tokens, attributes)+ metrics["genai_completion_tokens"].add(+ self._completion_tokens, attributes+ )+ metrics["genai_prompt_tokens"].add(+ self._prompt_tokens, attributes+ )metrics["genai_cost"].record(cost, attributes)except Exception as e:@@ -278,7 +291,8 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"))+ kwargs.get("model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,kwargs.get("top_p", 1.0))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,@@ -306,8 +320,12 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,# Set span attributes when tools is not passed to the function callif "tools" not in kwargs:# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),- pricing_info, response_dict.get('usage', {}).get('prompt_tokens', None),+ cost = get_chat_model_cost(kwargs.get(+ "model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"+ ),+ pricing_info,+ response_dict.get('usage', {}).get('prompt_tokens', None),response_dict.get('usage', {}).get('completion_tokens', None))span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,@@ -348,9 +366,14 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,# Set span attributes when tools is passed to the function callelif "tools" in kwargs:# Calculate cost of the operation- cost = get_chat_model_cost(kwargs.get("model", "meta-llama/Llama-3.3-70B-Instruct-Turbo"),- pricing_info, response_dict.get('usage').get('prompt_tokens'),+ cost = get_chat_model_cost(kwargs.get(+ "model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"+ ),+ pricing_info,+ response_dict.get('usage').get('prompt_tokens'),response_dict.get('usage').get('completion_tokens'))+span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={@@ -385,9 +408,12 @@ def async_completion(gen_ai_endpoint, version, environment, application_name,}metrics["genai_requests"].add(1, attributes)- metrics["genai_total_tokens"].add(response_dict.get('usage').get('total_tokens'), attributes)- metrics["genai_completion_tokens"].add(response_dict.get('usage').get('completion_tokens'), attributes)- metrics["genai_prompt_tokens"].add(response_dict.get('usage').get('prompt_tokens'), attributes)+ metrics["genai_total_tokens"].add(+ response_dict.get('usage').get('total_tokens'), attributes)+ metrics["genai_completion_tokens"].add(+ response_dict.get('usage').get('completion_tokens'), attributes)+ metrics["genai_prompt_tokens"].add(+ response_dict.get('usage').get('prompt_tokens'), attributes)metrics["genai_cost"].record(cost, attributes)# Return original response@@ -449,10 +475,14 @@ def async_image_generate(gen_ai_endpoint, version, environment, application_nameimage = "url"# Calculate cost of the operation- image_size = str(kwargs.get("height", 1024)) + "x" + str(kwargs.get("width", 1024))- cost = get_image_model_cost(kwargs.get("model", "black-forest-labs/FLUX.1-dev"),- pricing_info, image_size,+ image_size = str(kwargs.get("width", 1024)) + "x" + str(kwargs.get("height", 1024))+ cost_per_million = get_image_model_cost(kwargs.get(+ "model", "black-forest-labs/FLUX.1-dev"+ ),+ pricing_info, "1000000",kwargs.get("quality", "standard"))+ pixels = kwargs.get("width", 1024) * kwargs.get("height", 1024)+ cost = pixels / 1_000_000 * cost_per_millionfor items in response.data:# Set Span attributesdiff --git a/sdk/python/src/openlit/instrumentation/together/together.py b/sdk/python/src/openlit/instrumentation/together/together.pyindex 6a64c8e3..0f9d4625 100644--- a/sdk/python/src/openlit/instrumentation/together/together.py+++ b/sdk/python/src/openlit/instrumentation/together/together.py@@ -1,4 +1,4 @@-# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches, too-many-instance-attributes"""Module for monitoring Together calls."""@@ -8,7 +8,6 @@ from opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import TELEMETRY_SDK_NAMEfrom openlit.__helpers import (get_chat_model_cost,- general_tokens,get_image_model_cost,handle_exception,response_as_dict,@@ -141,8 +140,10 @@ def completion(gen_ai_endpoint, version, environment, application_name,self._span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,application_name)self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- self._kwargs.get("model",- "meta-llama/Llama-3.3-70B-Instruct-Turbo"))+ self._kwargs.get(+ "model",+ "meta-llama/Llama-3.3-70B-Instruct-Turbo"+ ))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,self._kwargs.get("user", ""))self._span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,@@ -323,7 +324,7 @@ def completion(gen_ai_endpoint, version, environment, application_name,"model","meta-llama/Llama-3.3-70B-Instruct-Turbo"),- pricing_info,+ pricing_info,response_dict.get('usage', {}).get('prompt_tokens', None),response_dict.get('usage', {}).get('completion_tokens', None))@@ -372,6 +373,7 @@ def completion(gen_ai_endpoint, version, environment, application_name,pricing_info,response_dict.get('usage').get('prompt_tokens'),response_dict.get('usage').get('completion_tokens'))+span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,attributes={@@ -473,10 +475,14 @@ def image_generate(gen_ai_endpoint, version, environment, application_name,image = "url"# Calculate cost of the operation- image_size = str(kwargs.get("height", 1024)) + "x" + str(kwargs.get("width", 1024))- cost = get_image_model_cost(kwargs.get("model", "black-forest-labs/FLUX.1-dev"),- pricing_info, image_size,+ image_size = str(kwargs.get("width", 1024)) + "x" + str(kwargs.get("height", 1024))+ cost_per_million = get_image_model_cost(kwargs.get(+ "model", "black-forest-labs/FLUX.1-dev"+ ),+ pricing_info, "1000000",kwargs.get("quality", "standard"))+ pixels = kwargs.get("width", 1024) * kwargs.get("height", 1024)+ cost = pixels / 1_000_000 * cost_per_millionfor items in response.data:# Set Span attributesdiff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txtindex 79e0ace8..44cf5d2c 100644--- a/sdk/python/tests/requirements.txt+++ b/sdk/python/tests/requirements.txt@@ -26,6 +26,7 @@ ai21controlflowassemblyaifirecrawl-py+together# Vector DBschromadbdiff --git a/sdk/python/tests/test_together.py b/sdk/python/tests/test_together.pyindex e0bc9b08..04255fbc 100644--- a/sdk/python/tests/test_together.py+++ b/sdk/python/tests/test_together.py@@ -13,7 +13,7 @@ prior to running these tests."""import pytest-from together import Together+from together import Together, AsyncTogetherimport openlit# Initialize synchronous Together client@@ -44,7 +44,7 @@ def test_sync_together_chat():max_tokens=1,stream=False,)- assert message.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'+ assert response.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'def test_sync_together_image():"""@@ -62,7 +62,7 @@ def test_sync_together_image():n=1,)- assert message.model == 'black-forest-labs/FLUX.1-dev'+ assert response.model == 'black-forest-labs/FLUX.1-dev'@pytest.mark.asyncioasync def test_async_together_chat():@@ -84,7 +84,7 @@ async def test_async_together_chat():max_tokens=1,stream=False,)- assert message.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'+ assert response.model == 'meta-llama/Llama-3.3-70B-Instruct-Turbo'@pytest.mark.asyncioasync def test_async_together_image():@@ -103,4 +103,4 @@ async def test_async_together_image():n=1,)- assert message.model == 'black-forest-labs/FLUX.1-dev'\ No newline at end of file+ assert response.model == 'black-forest-labs/FLUX.1-dev'