Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add Tracking Costs by using a finetuned model on Ollama Server (#368)

patcher9d81817123767
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index e25038c4..b8800799 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.18.0"
+version = "1.18.1"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
index f97294a3..dfa8f8cc 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
@@ -6,7 +6,11 @@ Module for monitoring Ollama API calls.
import logging
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
-from openlit.__helpers import handle_exception, general_tokens
+from openlit.__helpers import (
+ handle_exception,
+ general_tokens,
+ get_chat_model_cost,
+ get_embed_model_cost)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
@@ -90,10 +94,11 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,
formatted_messages.append(f"{role}: {content}")
prompt = "\n".join(formatted_messages)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(prompt)
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
@@ -219,11 +224,12 @@ def async_chat(gen_ai_endpoint, version, environment, application_name,
},
)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(prompt)
completion_tokens = response["eval_count"]
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
prompt_tokens)
@@ -331,10 +337,11 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,
# Handling exception ensure observability without disrupting operation
try:
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(kwargs.get("prompt", ""))
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
@@ -442,11 +449,12 @@ def async_generate(gen_ai_endpoint, version, environment, application_name,
},
)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = response["prompt_eval_count"]
completion_tokens = response["eval_count"]
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
prompt_tokens)
@@ -534,9 +542,10 @@ def async_embeddings(gen_ai_endpoint, version, environment, application_name,
response = await wrapped(*args, **kwargs)
try:
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
+ pricing_info, prompt_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
index 79071d3f..49a8ed8c 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
@@ -6,7 +6,12 @@ Module for monitoring Ollama API calls.
import logging
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
-from openlit.__helpers import handle_exception, general_tokens
+from openlit.__helpers import (
+ handle_exception,
+ general_tokens,
+ get_chat_model_cost,
+ get_embed_model_cost
+)
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
@@ -90,10 +95,11 @@ def chat(gen_ai_endpoint, version, environment, application_name,
formatted_messages.append(f"{role}: {content}")
prompt = "\n".join(formatted_messages)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(prompt)
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
@@ -219,11 +225,12 @@ def chat(gen_ai_endpoint, version, environment, application_name,
},
)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(prompt)
completion_tokens = response["eval_count"]
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
prompt_tokens)
@@ -331,10 +338,11 @@ def generate(gen_ai_endpoint, version, environment, application_name,
# Handling exception ensure observability without disrupting operation
try:
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(kwargs.get("prompt", ""))
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
@@ -442,11 +450,12 @@ def generate(gen_ai_endpoint, version, environment, application_name,
},
)
- # Calculate cost of the operation
- cost = 0
prompt_tokens = response["prompt_eval_count"]
completion_tokens = response["eval_count"]
total_tokens = prompt_tokens + completion_tokens
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "llama3"),
+ pricing_info, prompt_tokens, completion_tokens)
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
prompt_tokens)
@@ -534,9 +543,10 @@ def embeddings(gen_ai_endpoint, version, environment, application_name,
response = wrapped(*args, **kwargs)
try:
- # Calculate cost of the operation
- cost = 0
prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
+ pricing_info, prompt_tokens)
# Set Span attributes
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
diff --git a/sdk/python/tests/test_langchain.py b/sdk/python/tests/test_langchain.py.hold
similarity index 97%
rename from sdk/python/tests/test_langchain.py
rename to sdk/python/tests/test_langchain.py.hold
index b68be789..bc0bb7cc 100644
--- a/sdk/python/tests/test_langchain.py
+++ b/sdk/python/tests/test_langchain.py.hold
@@ -10,6 +10,7 @@ Note: Ensure the environment is properly configured for Langchain and OpenLIT mo
prior to running these tests.
"""
+import os
import bs4
from langchain import hub
from langchain_community.document_loaders import WebBaseLoader
@@ -40,6 +41,7 @@ def test_langchain():
Raises:
AssertionError: If the outcomes from the LangChain operations deviate from what is expected.
"""
+ os.environ["LANGCHAIN_TRACING_V2"] = "false"
docs = WebBaseLoader(
web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
diff --git a/sdk/python/tests/test_mistral.py b/sdk/python/tests/test_mistral.py.hold
similarity index 100%
rename from sdk/python/tests/test_mistral.py
rename to sdk/python/tests/test_mistral.py.hold
diff --git a/sdk/python/tests/test_ollama.py.hold b/sdk/python/tests/test_ollama.py.hold
new file mode 100644
index 00000000..26828fd4
--- /dev/null
+++ b/sdk/python/tests/test_ollama.py.hold
@@ -0,0 +1,39 @@
+import ollama
+import openlit
+
+openlit.init()
+
+
+stream = ollama.chat(
+ model='gemma2:2b',
+ messages=[{'role': 'user', 'content': 'Hello sync non-streaming, What is LLM observability?'}],
+ stream=False,
+)
+
+stream = ollama.chat(
+ model='gemma2:2b',
+ messages=[{'role': 'user', 'content': 'Hi sync streaming, What is LLM observability?'}],
+ stream=True,
+)
+
+for chunk in stream:
+ pass
+
+ollama.embeddings(model='gemma2:2b', prompt='Hi sync embedding, What is LLM observability?')
+
+
+import asyncio
+from ollama import AsyncClient
+
+async def chat():
+ message = {'role': 'user', 'content': 'Hi async non-streaming, What is LLM observability?'}
+ response = await AsyncClient().chat(model='gemma2:2b', messages=[message])
+
+ message = {'role': 'user', 'content': 'Hi async streaming, What is LLM observability?'}
+ async for part in await AsyncClient().chat(model='gemma2:2b', messages=[message], stream=True):
+ pass
+
+ await AsyncClient().embeddings(model='gemma2:2b', prompt='Hi async embedding, What is LLM observability?')
+
+
+asyncio.run(chat())
\ No newline at end of file