DSPy: The framework for programming—not prompting—language models
Python
clean up for tasks
klopsahlong6ae4ad400f9a
diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.pyindex 47bcc80ae..13ddc6263 100644--- a/testing/optimizer_tester.py+++ b/testing/optimizer_tester.py@@ -20,15 +20,7 @@ from tasks.hover import HoverRetrieveDiscretefrom tasks.iris_typo import IrisTypoClassifierTaskfrom tasks.iris import IrisClassifierTask-# datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "biodex", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]-# datasets = ["iris"] # working-# datasets = ["hotpotqa_conditional"] # working-# datasets = ["iris_typo"] # working-datasets = ["hotpotqa"] # <- not working-# datasets = ["gsm8k"] # working-# datasets = ["biodex"] # not working-# datasets = ["heart_disease"] # working-# datasets = ["hover_retrieve_discrete"] # not working, need to debug+datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "biodex", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]class OptimizerTester:def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,@@ -108,8 +100,6 @@ class OptimizerTester:ds = HotPotQAConditionalTask()elif dataset == "gsm8k":ds = GSM8KTask()- elif dataset == "biodex":- ds = BioDexTask()elif dataset == "tweet":ds = TweetTask()elif dataset == "heart_disease":diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.pyindex 277aad8ca..fe8136b96 100644--- a/testing/tasks/gsm8k.py+++ b/testing/tasks/gsm8k.py@@ -62,10 +62,6 @@ class GSM8KTask(BaseTask):devset = [dspy.Example(**x).with_inputs('question') for x in devset]testset = [dspy.Example(**x).with_inputs('question') for x in testset]- # print(f"Trainset size: {len(trainset)}")- # print(f"Devset size: {len(devset)}")- # print(f"Testset size: {len(testset)}")-self.trainset= trainsetself.devset = devsetself.testset = testsetdiff --git a/testing/tasks/heart_disease.py b/testing/tasks/heart_disease.pyindex b25d8657d..9d3638208 100644--- a/testing/tasks/heart_disease.py+++ b/testing/tasks/heart_disease.py@@ -70,10 +70,8 @@ class Classify(dspy.Module):thalach=thalach, exang=exang, oldpeak=oldpeak, slope=slope, ca=ca, thal=thal)opinions = [c(**kwargs) for c in self.classify]- try:- opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]- except:- breakpoint()+ opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]+opinions = [f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}." for reason, answer in opinions]return self.vote(context=opinions, **kwargs)diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.pyindex eb3873e0e..5c62b5dba 100644--- a/testing/tasks/hotpotqa.py+++ b/testing/tasks/hotpotqa.py@@ -10,7 +10,6 @@ class MultiHop(dspy.Module):self.retrieve = dspy.Retrieve(k=passages_per_hop)self.generate_query = dspy.ChainOfThought("context ,question->search_query")self.generate_answer = dspy.ChainOfThought("context ,question->answer")- # self.generate_answer = dspy.ChainOfThought(AdversarialAnswer)def forward(self, question):context = []@@ -26,16 +25,11 @@ class MultiHop(dspy.Module):class HotPotQATask(BaseTask):def __init__(self):# Load and configure the datasets.- # hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=1000)- # hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, keep_details=True)hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023)self.trainset = [x.with_inputs("question") for x in hotpot_dataset.train]- # self.devset = [x.with_inputs("question") for x in hotpot_dataset.dev]self.testset = [x.with_inputs("question") for x in hotpot_dataset.dev]- #self.testset = self.testset[:500]- # self.testset = self.testset[:1000]# Set up metricsNUM_THREADS=16@@ -63,7 +57,7 @@ class HotPotQATask(BaseTask):def get_program(self):return MultiHop(passages_per_hop=3- ) # TODO: make it so we can specify # of passages+ )def get_metric(self):return self.metricdiff --git a/testing/tasks/hotpotqa_conditional.py b/testing/tasks/hotpotqa_conditional.pyindex f4eada09c..c9466a834 100644--- a/testing/tasks/hotpotqa_conditional.py+++ b/testing/tasks/hotpotqa_conditional.py@@ -76,7 +76,6 @@ class MultiHop(dspy.Module):self.retrieve = dspy.Retrieve(k=passages_per_hop)self.generate_query = dspy.ChainOfThought("context ,question->search_query")self.generate_answer = dspy.ChainOfThought("context ,question->answer")- # self.generate_answer = dspy.ChainOfThought(AdversarialAnswer)def forward(self, question):context = []@@ -129,10 +128,6 @@ class HotPotQAConditionalTask(BaseTask):hotpotqa_dev = pd.read_csv(hotpotqa_dev_path)hotpotqa_test = pd.read_csv(hotpotqa_test_path)- # hotpotqa_train = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_train.csv")- # hotpotqa_dev = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_dev.csv")- # hotpotqa_test = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_test.csv")-combined_train = pd.concat([hotpotqa_train, hotpotqa_dev], ignore_index=True)# Load and configure the datasets.@@ -162,9 +157,6 @@ class HotPotQAConditionalTask(BaseTask):def get_default_max_labeled_demos(self):return 0-- def get_handwritten_prompts(self):- return []def get_max_tokens(self):return 600diff --git a/testing/tasks/hover.py b/testing/tasks/hover.pyindex 619f6d91b..c7e4c1161 100644--- a/testing/tasks/hover.py+++ b/testing/tasks/hover.py@@ -15,7 +15,6 @@ def count_unique_docs(example):def discrete_retrieval_eval(example, pred, trace=None):gold_titles = set(map(dspy.evaluate.normalize_text, [doc['key'] for doc in example['supporting_facts']]))found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.retrieved_docs]))- # print(f"example {example} | gold titles {gold_titles} | found titles {found_titles} | Is subset: {gold_titles.issubset(found_titles)}")return gold_titles.issubset(found_titles)class RetrieveMultiHop(dspy.Module):diff --git a/testing/tasks/iris.py b/testing/tasks/iris.pyindex dce4fd0d7..64e0fcab6 100644--- a/testing/tasks/iris.py+++ b/testing/tasks/iris.py@@ -6,7 +6,6 @@ import dspyclass Sig(dspy.Signature):"Given the petal and sepal dimensions in cm, predict the iris species."- # "Given the petal and sepal dimensions in cm, predict the iris species. If petal_width < 1.0, setosa. Otherwise, if petal_width < 1.65, versicolor. Otherwise, virginica."petal_length = dspy.InputField()petal_width = dspy.InputField()diff --git a/testing/tasks/iris_typo.py b/testing/tasks/iris_typo.pyindex 64318a81c..ee0e6b3a6 100644--- a/testing/tasks/iris_typo.py+++ b/testing/tasks/iris_typo.py@@ -31,7 +31,6 @@ class IrisTypoClassifierTask(BaseTask):fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]random.Random(0).shuffle(fullset)- # self.trainset, self.devset, self.testset = fullset[:25], fullset[20:75], fullset[75:]self.trainset, self.testset = fullset[:75], fullset[75:]# Set up metrics@@ -66,7 +65,6 @@ class IrisTypoClassifierTask(BaseTask):return self.testsetdef get_devset(self, TRAIN_NUM=None, DEV_NUM=None):- # return self.devsetreturn self.trainsetdef get_evalset(self, EVAL_NUM=None):diff --git a/testing/tasks/scone.py b/testing/tasks/scone.pyindex fc720004a..d0c5f5193 100644--- a/testing/tasks/scone.py+++ b/testing/tasks/scone.py@@ -64,7 +64,7 @@ class ScoNeTask(BaseTask):random.shuffle(all_train)# 1000 random train, 500 random dev:- self.trainset, self.devset = all_train[: 1000], all_train[1000: 1500]+ self.trainset, self.testset = all_train[: 1000], all_train[1000: 1500]metric_EM = dspy.evaluate.answer_exact_matchself.metric = metric_EMdiff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.pyindex 83f759061..3e6c19a8c 100644--- a/testing/tasks/tweet.py+++ b/testing/tasks/tweet.py@@ -55,7 +55,6 @@ def load_models():openai.api_key = os.environ.get('OPENAI_API_KEY')openai.api_base = os.environ.get('OPENAI_API_BASE')gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')- # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')retrieve = dspy.Retrieve(k=5)return gpt4T, retrieve@@ -96,10 +95,10 @@ class TweetTask(BaseTask):# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]- self.devset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]+ self.testset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]self.trainset = [x.with_inputs('question') for x in dataset.train]- self.devset = [x.with_inputs('question') for x in dataset.dev]+ self.testset = [x.with_inputs('question') for x in dataset.dev]self.metric = metricdiff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.pyindex a37e6bd43..a1c6f0464 100644--- a/testing/tasks/tweet_metric.py+++ b/testing/tasks/tweet_metric.py@@ -116,7 +116,7 @@ class TweetMetricTask(BaseTask):trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]self.trainset = []- self.devset = []+ self.testset = []gpt3T, gpt4T, retrieve = load_models()@@ -137,7 +137,7 @@ class TweetMetricTask(BaseTask):tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answerexample = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}- self.devset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))+ self.testset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))self.metric = metric