Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Remove the dependency on tiktoken (#646)

Patcher2e7f353320a1
diff --git a/.github/workflows/test-pypi-release.yml b/.github/workflows/test-pypi-release.yml
index b2d0e812..3395b462 100644
--- a/.github/workflows/test-pypi-release.yml
+++ b/.github/workflows/test-pypi-release.yml
@@ -45,7 +45,6 @@ jobs:
[tool.poetry.dependencies]
python = "^3.7.1"
requests = "^2.26.0"
- tiktoken = "^0.1.1"
opentelemetry-api = "^1.24.0"
opentelemetry-sdk = "^1.24.0"
opentelemetry-exporter-otlp = "^1.24.0"
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 0319aefc..803f1884 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.33.12"
+version = "1.33.13"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
@@ -13,7 +13,6 @@ keywords = ["OpenTelemetry", "otel", "otlp","llm", "tracing", "openai", "anthrop
python = "^3.7.1"
requests = "^2.26.0"
schedule = "^1.2.2"
-tiktoken = "^0.7.0"
pydantic = "^2.0.0"
xmltodict = "^0.13.0"
boto3 = "^1.34.0"
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index 88fdb869..7579eecd 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -7,8 +7,8 @@ import json
import logging
from urllib.parse import urlparse
from typing import Any, Dict, List, Tuple
+import math
import requests
-import tiktoken
from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
from opentelemetry.trace import Status, StatusCode
from opentelemetry._events import Event
@@ -45,28 +45,12 @@ def get_env_variable(name, arg_value, error_message):
raise RuntimeError(error_message)
return value
-def openai_tokens(text, model):
- """
- Calculate the number of tokens a given text would take up for a specified model.
- """
-
- try:
- encoding = tiktoken.encoding_for_model(model)
- except:
- encoding = tiktoken.get_encoding("cl100k_base")
-
- num_tokens = len(encoding.encode(text))
- return num_tokens
-
def general_tokens(text):
"""
Calculate the number of tokens a given text would take up.
"""
- encoding = tiktoken.get_encoding('gpt2')
-
- num_tokens = len(encoding.encode(text))
- return num_tokens
+ return math.ceil(len(text) / 2)
def get_chat_model_cost(model, pricing_info, prompt_tokens, completion_tokens):
"""
diff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
index 8ddb7846..af5d91b6 100644
--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py
+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
@@ -11,7 +11,7 @@ from openlit.__helpers import (
get_embed_model_cost,
get_audio_model_cost,
get_image_model_cost,
- openai_tokens,
+ general_tokens,
handle_exception,
response_as_dict,
calculate_ttft,
@@ -151,10 +151,8 @@ def async_chat_completions(version, environment, application_name,
request_model = self._kwargs.get("model", "gpt-4o")
# Calculate tokens using input prompt and aggregated response
- input_tokens = openai_tokens(prompt,
- request_model)
- output_tokens = openai_tokens(self._llmresponse,
- request_model)
+ input_tokens = general_tokens(prompt)
+ output_tokens = general_tokens(self._llmresponse)
# Calculate cost of the operation
cost = get_chat_model_cost(request_model,
diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.py
index 3251b053..69f608d6 100644
--- a/sdk/python/src/openlit/instrumentation/openai/openai.py
+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py
@@ -11,7 +11,7 @@ from openlit.__helpers import (
get_embed_model_cost,
get_audio_model_cost,
get_image_model_cost,
- openai_tokens,
+ general_tokens,
handle_exception,
response_as_dict,
calculate_ttft,
@@ -151,10 +151,8 @@ def chat_completions(version, environment, application_name,
request_model = self._kwargs.get("model", "gpt-4o")
# Calculate tokens using input prompt and aggregated response
- input_tokens = openai_tokens(prompt,
- request_model)
- output_tokens = openai_tokens(self._llmresponse,
- request_model)
+ input_tokens = general_tokens(prompt)
+ output_tokens = general_tokens(self._llmresponse)
# Calculate cost of the operation
cost = get_chat_model_cost(request_model,
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index f074df03..f20eb254 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -45,7 +45,6 @@ xmltodict
# Utilities
bs4
-tiktoken
tensorflow
tf_keras
aiohttp