DSPy: The framework for programming—not prompting—language models
Python
move tiktoken dependency
Isaac Millere2389b4428c9
diff --git a/dsp/modules/openai.py b/dsp/modules/openai.pyindex 1a92abf66..93238cffb 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/openai.py@@ -5,7 +5,6 @@ from typing import Any, Optional, Literal, Unionimport openaifrom dsp.modules.lm import FinetunableLMfrom dsp.modules.gpt3 import GPT3-import tiktokenfrom collections import defaultdict@@ -58,10 +57,11 @@ def openai_data_validation(dataset: dict[str, Any]) -> Union[dict[str, Any], Nonprint("No errors found")-encoding = tiktoken.get_encoding("cl100k_base")-def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):+ import tiktoken+ encoding = tiktoken.get_encoding("cl100k_base")+num_tokens = 0for message in messages:num_tokens += tokens_per_message@@ -74,6 +74,9 @@ def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):def num_assistant_tokens_from_messages(messages):+ import tiktoken+ encoding = tiktoken.get_encoding("cl100k_base")+num_tokens = 0for message in messages:if message["role"] == "assistant":