Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Add Tracking Costs by using a finetuned model on Ollama Server (#368)
patcher9d81817123767
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex e25038c4..b8800799 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.18.0"+version = "1.18.1"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.pyindex f97294a3..dfa8f8cc 100644--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py@@ -6,7 +6,11 @@ Module for monitoring Ollama API calls.import loggingfrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import handle_exception, general_tokens+from openlit.__helpers import (+ handle_exception,+ general_tokens,+ get_chat_model_cost,+ get_embed_model_cost)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operations@@ -90,10 +94,11 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(prompt)total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")@@ -219,11 +224,12 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,},)- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(prompt)completion_tokens = response["eval_count"]total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,prompt_tokens)@@ -331,10 +337,11 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,# Handling exception ensure observability without disrupting operationtry:- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(kwargs.get("prompt", ""))total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")@@ -442,11 +449,12 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,},)- # Calculate cost of the operation- cost = 0prompt_tokens = response["prompt_eval_count"]completion_tokens = response["eval_count"]total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,prompt_tokens)@@ -534,9 +542,10 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,response = await wrapped(*args, **kwargs)try:- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(kwargs.get('prompt', ""))+ # Calculate cost of the operation+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),+ pricing_info, prompt_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.pyindex 79071d3f..49a8ed8c 100644--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py@@ -6,7 +6,12 @@ Module for monitoring Ollama API calls.import loggingfrom opentelemetry.trace import SpanKind, Status, StatusCodefrom opentelemetry.sdk.resources import TELEMETRY_SDK_NAME-from openlit.__helpers import handle_exception, general_tokens+from openlit.__helpers import (+ handle_exception,+ general_tokens,+ get_chat_model_cost,+ get_embed_model_cost+)from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operations@@ -90,10 +95,11 @@ def chat(gen_ai_endpoint, version, environment, application_name,formatted_messages.append(f"{role}: {content}")prompt = "\n".join(formatted_messages)- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(prompt)total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")@@ -219,11 +225,12 @@ def chat(gen_ai_endpoint, version, environment, application_name,},)- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(prompt)completion_tokens = response["eval_count"]total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,prompt_tokens)@@ -331,10 +338,11 @@ def generate(gen_ai_endpoint, version, environment, application_name,# Handling exception ensure observability without disrupting operationtry:- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(kwargs.get("prompt", ""))total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")@@ -442,11 +450,12 @@ def generate(gen_ai_endpoint, version, environment, application_name,},)- # Calculate cost of the operation- cost = 0prompt_tokens = response["prompt_eval_count"]completion_tokens = response["eval_count"]total_tokens = prompt_tokens + completion_tokens+ # Calculate cost of the operation+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),+ pricing_info, prompt_tokens, completion_tokens)span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,prompt_tokens)@@ -534,9 +543,10 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,response = wrapped(*args, **kwargs)try:- # Calculate cost of the operation- cost = 0prompt_tokens = general_tokens(kwargs.get('prompt', ""))+ # Calculate cost of the operation+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),+ pricing_info, prompt_tokens)# Set Span attributesspan.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,diff --git a/sdk/python/tests/test_langchain.py b/sdk/python/tests/test_langchain.py.holdsimilarity index 97%rename from sdk/python/tests/test_langchain.pyrename to sdk/python/tests/test_langchain.py.holdindex b68be789..bc0bb7cc 100644--- a/sdk/python/tests/test_langchain.py+++ b/sdk/python/tests/test_langchain.py.hold@@ -10,6 +10,7 @@ Note: Ensure the environment is properly configured for Langchain and OpenLIT moprior to running these tests."""+import osimport bs4from langchain import hubfrom langchain_community.document_loaders import WebBaseLoader@@ -40,6 +41,7 @@ def test_langchain():Raises:AssertionError: If the outcomes from the LangChain operations deviate from what is expected."""+ os.environ["LANGCHAIN_TRACING_V2"] = "false"docs = WebBaseLoader(web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),diff --git a/sdk/python/tests/test_mistral.py b/sdk/python/tests/test_mistral.py.holdsimilarity index 100%rename from sdk/python/tests/test_mistral.pyrename to sdk/python/tests/test_mistral.py.holddiff --git a/sdk/python/tests/test_ollama.py.hold b/sdk/python/tests/test_ollama.py.holdnew file mode 100644index 00000000..26828fd4--- /dev/null+++ b/sdk/python/tests/test_ollama.py.hold@@ -0,0 +1,39 @@+import ollama+import openlit++openlit.init()+++stream = ollama.chat(+ model='gemma2:2b',+ messages=[{'role': 'user', 'content': 'Hello sync non-streaming, What is LLM observability?'}],+ stream=False,+)++stream = ollama.chat(+ model='gemma2:2b',+ messages=[{'role': 'user', 'content': 'Hi sync streaming, What is LLM observability?'}],+ stream=True,+)++for chunk in stream:+ pass++ollama.embeddings(model='gemma2:2b', prompt='Hi sync embedding, What is LLM observability?')+++import asyncio+from ollama import AsyncClient++async def chat():+ message = {'role': 'user', 'content': 'Hi async non-streaming, What is LLM observability?'}+ response = await AsyncClient().chat(model='gemma2:2b', messages=[message])++ message = {'role': 'user', 'content': 'Hi async streaming, What is LLM observability?'}+ async for part in await AsyncClient().chat(model='gemma2:2b', messages=[message], stream=True):+ pass++ await AsyncClient().embeddings(model='gemma2:2b', prompt='Hi async embedding, What is LLM observability?')+++asyncio.run(chat())\ No newline at end of file