Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

clean up for tasks

klopsahlong6ae4ad400f9a
diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.py
index 47bcc80ae..13ddc6263 100644
--- a/testing/optimizer_tester.py
+++ b/testing/optimizer_tester.py
@@ -20,15 +20,7 @@ from tasks.hover import HoverRetrieveDiscrete
from tasks.iris_typo import IrisTypoClassifierTask
from tasks.iris import IrisClassifierTask
-# datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "biodex", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]
-# datasets = ["iris"] # working
-# datasets = ["hotpotqa_conditional"] # working
-# datasets = ["iris_typo"] # working
-datasets = ["hotpotqa"] # <- not working
-# datasets = ["gsm8k"] # working
-# datasets = ["biodex"] # not working
-# datasets = ["heart_disease"] # working
-# datasets = ["hover_retrieve_discrete"] # not working, need to debug
+datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "biodex", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]
class OptimizerTester:
def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,
@@ -108,8 +100,6 @@ class OptimizerTester:
ds = HotPotQAConditionalTask()
elif dataset == "gsm8k":
ds = GSM8KTask()
- elif dataset == "biodex":
- ds = BioDexTask()
elif dataset == "tweet":
ds = TweetTask()
elif dataset == "heart_disease":
diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.py
index 277aad8ca..fe8136b96 100644
--- a/testing/tasks/gsm8k.py
+++ b/testing/tasks/gsm8k.py
@@ -62,10 +62,6 @@ class GSM8KTask(BaseTask):
devset = [dspy.Example(**x).with_inputs('question') for x in devset]
testset = [dspy.Example(**x).with_inputs('question') for x in testset]
- # print(f"Trainset size: {len(trainset)}")
- # print(f"Devset size: {len(devset)}")
- # print(f"Testset size: {len(testset)}")
-
self.trainset= trainset
self.devset = devset
self.testset = testset
diff --git a/testing/tasks/heart_disease.py b/testing/tasks/heart_disease.py
index b25d8657d..9d3638208 100644
--- a/testing/tasks/heart_disease.py
+++ b/testing/tasks/heart_disease.py
@@ -70,10 +70,8 @@ class Classify(dspy.Module):
thalach=thalach, exang=exang, oldpeak=oldpeak, slope=slope, ca=ca, thal=thal)
opinions = [c(**kwargs) for c in self.classify]
- try:
- opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]
- except:
- breakpoint()
+ opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]
+
opinions = [f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}." for reason, answer in opinions]
return self.vote(context=opinions, **kwargs)
diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.py
index eb3873e0e..5c62b5dba 100644
--- a/testing/tasks/hotpotqa.py
+++ b/testing/tasks/hotpotqa.py
@@ -10,7 +10,6 @@ class MultiHop(dspy.Module):
self.retrieve = dspy.Retrieve(k=passages_per_hop)
self.generate_query = dspy.ChainOfThought("context ,question->search_query")
self.generate_answer = dspy.ChainOfThought("context ,question->answer")
- # self.generate_answer = dspy.ChainOfThought(AdversarialAnswer)
def forward(self, question):
context = []
@@ -26,16 +25,11 @@ class MultiHop(dspy.Module):
class HotPotQATask(BaseTask):
def __init__(self):
# Load and configure the datasets.
- # hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=1000)
- # hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, keep_details=True)
hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023)
self.trainset = [x.with_inputs("question") for x in hotpot_dataset.train]
- # self.devset = [x.with_inputs("question") for x in hotpot_dataset.dev]
self.testset = [x.with_inputs("question") for x in hotpot_dataset.dev]
- #self.testset = self.testset[:500]
- # self.testset = self.testset[:1000]
# Set up metrics
NUM_THREADS=16
@@ -63,7 +57,7 @@ class HotPotQATask(BaseTask):
def get_program(self):
return MultiHop(
passages_per_hop=3
- ) # TODO: make it so we can specify # of passages
+ )
def get_metric(self):
return self.metric
diff --git a/testing/tasks/hotpotqa_conditional.py b/testing/tasks/hotpotqa_conditional.py
index f4eada09c..c9466a834 100644
--- a/testing/tasks/hotpotqa_conditional.py
+++ b/testing/tasks/hotpotqa_conditional.py
@@ -76,7 +76,6 @@ class MultiHop(dspy.Module):
self.retrieve = dspy.Retrieve(k=passages_per_hop)
self.generate_query = dspy.ChainOfThought("context ,question->search_query")
self.generate_answer = dspy.ChainOfThought("context ,question->answer")
- # self.generate_answer = dspy.ChainOfThought(AdversarialAnswer)
def forward(self, question):
context = []
@@ -129,10 +128,6 @@ class HotPotQAConditionalTask(BaseTask):
hotpotqa_dev = pd.read_csv(hotpotqa_dev_path)
hotpotqa_test = pd.read_csv(hotpotqa_test_path)
- # hotpotqa_train = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_train.csv")
- # hotpotqa_dev = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_dev.csv")
- # hotpotqa_test = pd.read_csv("../datasets/hotpotqa_conditional/hotpot_test.csv")
-
combined_train = pd.concat([hotpotqa_train, hotpotqa_dev], ignore_index=True)
# Load and configure the datasets.
@@ -162,9 +157,6 @@ class HotPotQAConditionalTask(BaseTask):
def get_default_max_labeled_demos(self):
return 0
-
- def get_handwritten_prompts(self):
- return []
def get_max_tokens(self):
return 600
diff --git a/testing/tasks/hover.py b/testing/tasks/hover.py
index 619f6d91b..c7e4c1161 100644
--- a/testing/tasks/hover.py
+++ b/testing/tasks/hover.py
@@ -15,7 +15,6 @@ def count_unique_docs(example):
def discrete_retrieval_eval(example, pred, trace=None):
gold_titles = set(map(dspy.evaluate.normalize_text, [doc['key'] for doc in example['supporting_facts']]))
found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.retrieved_docs]))
- # print(f"example {example} | gold titles {gold_titles} | found titles {found_titles} | Is subset: {gold_titles.issubset(found_titles)}")
return gold_titles.issubset(found_titles)
class RetrieveMultiHop(dspy.Module):
diff --git a/testing/tasks/iris.py b/testing/tasks/iris.py
index dce4fd0d7..64e0fcab6 100644
--- a/testing/tasks/iris.py
+++ b/testing/tasks/iris.py
@@ -6,7 +6,6 @@ import dspy
class Sig(dspy.Signature):
"Given the petal and sepal dimensions in cm, predict the iris species."
- # "Given the petal and sepal dimensions in cm, predict the iris species. If petal_width < 1.0, setosa. Otherwise, if petal_width < 1.65, versicolor. Otherwise, virginica."
petal_length = dspy.InputField()
petal_width = dspy.InputField()
diff --git a/testing/tasks/iris_typo.py b/testing/tasks/iris_typo.py
index 64318a81c..ee0e6b3a6 100644
--- a/testing/tasks/iris_typo.py
+++ b/testing/tasks/iris_typo.py
@@ -31,7 +31,6 @@ class IrisTypoClassifierTask(BaseTask):
fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]
random.Random(0).shuffle(fullset)
- # self.trainset, self.devset, self.testset = fullset[:25], fullset[20:75], fullset[75:]
self.trainset, self.testset = fullset[:75], fullset[75:]
# Set up metrics
@@ -66,7 +65,6 @@ class IrisTypoClassifierTask(BaseTask):
return self.testset
def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):
- # return self.devset
return self.trainset
def get_evalset(self, EVAL_NUM=None):
diff --git a/testing/tasks/scone.py b/testing/tasks/scone.py
index fc720004a..d0c5f5193 100644
--- a/testing/tasks/scone.py
+++ b/testing/tasks/scone.py
@@ -64,7 +64,7 @@ class ScoNeTask(BaseTask):
random.shuffle(all_train)
# 1000 random train, 500 random dev:
- self.trainset, self.devset = all_train[: 1000], all_train[1000: 1500]
+ self.trainset, self.testset = all_train[: 1000], all_train[1000: 1500]
metric_EM = dspy.evaluate.answer_exact_match
self.metric = metric_EM
diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.py
index 83f759061..3e6c19a8c 100644
--- a/testing/tasks/tweet.py
+++ b/testing/tasks/tweet.py
@@ -55,7 +55,6 @@ def load_models():
openai.api_key = os.environ.get('OPENAI_API_KEY')
openai.api_base = os.environ.get('OPENAI_API_BASE')
gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')
- # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')
retrieve = dspy.Retrieve(k=5)
return gpt4T, retrieve
@@ -96,10 +95,10 @@ class TweetTask(BaseTask):
# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
- self.devset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
+ self.testset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
self.trainset = [x.with_inputs('question') for x in dataset.train]
- self.devset = [x.with_inputs('question') for x in dataset.dev]
+ self.testset = [x.with_inputs('question') for x in dataset.dev]
self.metric = metric
diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.py
index a37e6bd43..a1c6f0464 100644
--- a/testing/tasks/tweet_metric.py
+++ b/testing/tasks/tweet_metric.py
@@ -116,7 +116,7 @@ class TweetMetricTask(BaseTask):
trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
self.trainset = []
- self.devset = []
+ self.testset = []
gpt3T, gpt4T, retrieve = load_models()
@@ -137,7 +137,7 @@ class TweetMetricTask(BaseTask):
tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer
example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}
- self.devset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))
+ self.testset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))
self.metric = metric