Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

formatting updates

klopsahlongc223c3be48ed
diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.py
index 09cae6b62..841512472 100644
--- a/testing/optimizer_tester.py
+++ b/testing/optimizer_tester.py
@@ -20,14 +20,38 @@ from tasks.hover import HoverRetrieveDiscrete
from tasks.iris_typo import IrisTypoClassifierTask
from tasks.iris import IrisClassifierTask
-datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]
+datasets = [
+ "scone",
+ "hotpotqa",
+ "hotpotqa_conditional",
+ "gsm8k",
+ "tweet",
+ "heart_disease",
+ "iris",
+ "iris_typo",
+ "hover_retrieve_discrete",
+ "tweet_metric",
+]
+
class OptimizerTester:
- def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,
- num_threads = 10, default_breadth = 10, default_depth = 3, default_temperature = 1.1,
- prompt_model_name = "gpt-3.5-turbo-1106", task_model_name = "meta-llama/Llama-2-13b-chat-hf",
- prompt_model = None, task_model = None, max_errors=100,
- colbert_v2_endpoint = "http://20.102.90.50:2017/wiki17_abstracts"):
+ def __init__(
+ self,
+ datasets=datasets,
+ default_train_num=200,
+ default_dev_num=100,
+ default_test_num=200,
+ num_threads=10,
+ default_breadth=10,
+ default_depth=3,
+ default_temperature=1.1,
+ prompt_model_name="gpt-3.5-turbo-1106",
+ task_model_name="meta-llama/Llama-2-13b-chat-hf",
+ prompt_model=None,
+ task_model=None,
+ max_errors=100,
+ colbert_v2_endpoint="http://20.102.90.50:2017/wiki17_abstracts",
+ ):
self.datasets = datasets
self.TRAIN_NUM = default_train_num
self.DEV_NUM = default_dev_num
@@ -43,18 +67,24 @@ class OptimizerTester:
load_dotenv() # This will load the .env file's variables
- openai.api_key = os.environ.get('OPENAI_API_KEY')
- openai.api_base = os.environ.get('OPENAI_API_BASE')
+ openai.api_key = os.environ.get("OPENAI_API_KEY")
+ openai.api_base = os.environ.get("OPENAI_API_BASE")
# Prompt gen model
if not prompt_model:
- self.prompt_model = dspy.OpenAI(model=self.PROMPT_MODEL_NAME, max_tokens=700)
+ self.prompt_model = dspy.OpenAI(
+ model=self.PROMPT_MODEL_NAME, max_tokens=700
+ )
else:
self.prompt_model = prompt_model
# Task model
if not task_model:
- self.task_model = dspy.HFClientTGI(model=self.TASK_MODEL_NAME, port=[7140, 7141, 7142, 7143], max_tokens=150)
+ self.task_model = dspy.HFClientTGI(
+ model=self.TASK_MODEL_NAME,
+ port=[7140, 7141, 7142, 7143],
+ max_tokens=150,
+ )
else:
self.task_model = task_model
self.colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)
@@ -65,21 +95,47 @@ class OptimizerTester:
# Ensure the output directory exists
os.makedirs(folder_name, exist_ok=True)
file_path = os.path.join(folder_name, file_name)
-
+
# Check if file exists to determine if headers should be written
file_exists = os.path.isfile(file_path)
- with open(file_path, mode='a', newline='', encoding='utf-8') as file:
+ with open(file_path, mode="a", newline="", encoding="utf-8") as file:
writer = csv.writer(file)
- headers = ['test_name', 'train_score', 'dev_score', 'test_score', 'run_time (sec)', 'train_size', 'dev_size', 'test_size', 'task_name', 'signature_optimized', 'prompt_model_name', 'task_model_name', 'breadth', 'depth', 'meta_prompt_style', 'fewshot_before', 'fewshot_after', 'temperature', 'fewshot_candidates_num', 'max_bootstrapped_demos', 'bootstrapping', 'view_data', 'optimizer_log_dir', 'additional_notes', 'misc']
-
+ headers = [
+ "test_name",
+ "train_score",
+ "dev_score",
+ "test_score",
+ "run_time (sec)",
+ "train_size",
+ "dev_size",
+ "test_size",
+ "task_name",
+ "signature_optimized",
+ "prompt_model_name",
+ "task_model_name",
+ "breadth",
+ "depth",
+ "meta_prompt_style",
+ "fewshot_before",
+ "fewshot_after",
+ "temperature",
+ "fewshot_candidates_num",
+ "max_bootstrapped_demos",
+ "bootstrapping",
+ "view_data",
+ "optimizer_log_dir",
+ "additional_notes",
+ "misc",
+ ]
+
# Write headers if the file is being created
if not file_exists:
writer.writerow(headers)
-
+
# Format the data
- formatted_data = ['NA'] * len(headers)
+ formatted_data = ["NA"] * len(headers)
formatted_data[-1] = ""
for key in data.keys():
if key in headers:
@@ -89,7 +145,7 @@ class OptimizerTester:
# Write the data
writer.writerow(formatted_data)
- def load_dataset(self,dataset):
+ def load_dataset(self, dataset):
ds = None
dataset = dataset.lower()
if dataset == "scone":
@@ -114,9 +170,11 @@ class OptimizerTester:
ds = TweetMetricTask()
else:
raise ValueError("Invalid dataset name.")
- ds.set_splits(TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, TEST_NUM=self.TEST_NUM)
+ ds.set_splits(
+ TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, TEST_NUM=self.TEST_NUM
+ )
return ds
-
+
# Computes baseline results for a given dataset
def test_baseline(self, datasets=datasets, test_name="baseline"):
for dataset in datasets:
@@ -124,9 +182,27 @@ class OptimizerTester:
task = self.load_dataset(dataset)
dspy.settings.lm.max_tokens = task.get_max_tokens()
- evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)
- evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)
- evaluate_test = Evaluate(devset=task.get_testset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)
+ evaluate_train = Evaluate(
+ devset=task.get_trainset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
+ evaluate_dev = Evaluate(
+ devset=task.get_devset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
+ evaluate_test = Evaluate(
+ devset=task.get_testset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
default_program = task.get_program()
# Evaluate the default program
@@ -138,55 +214,97 @@ class OptimizerTester:
default_results_test = evaluate_test(default_program)
# Write the results to a csv
- self.write_to_csv('outputs', 'results.csv', {
- 'test_name': dataset + "_" + test_name,
- 'train_score': default_results_train,
- 'dev_score': default_results_dev,
- 'test_score': default_results_test,
- 'run_time (sec)': 0,
- 'train_size': len(task.get_trainset()),
- 'dev_size': len(task.get_devset()),
- 'test_size': len(task.get_testset()),
- 'task_name': dataset,
- 'signature_optimized': False,
- 'prompt_model_name': self.PROMPT_MODEL_NAME,
- 'task_model_name': self.TASK_MODEL_NAME,
- 'breadth': 'NA',
- 'depth': 'NA',
- 'meta_prompt_style': 'default',
- 'fewshot_before': False,
- 'fewshot_after': False,
- 'temperature': self.TEMPERATURE,
- 'fewshot_candidates_num': 0,
- 'max_bootstrapped_demos': 0,
- 'bootstrapping': False,
- 'view_data': False,
- 'optimizer_log_dir': 'NA',
- 'additional_notes': '',
- 'misc': '',
- })
-
- def test_optimizer_default(self, optimizer_function, datasets=datasets, test_name="default"):
+ self.write_to_csv(
+ "outputs",
+ "results.csv",
+ {
+ "test_name": dataset + "_" + test_name,
+ "train_score": default_results_train,
+ "dev_score": default_results_dev,
+ "test_score": default_results_test,
+ "run_time (sec)": 0,
+ "train_size": len(task.get_trainset()),
+ "dev_size": len(task.get_devset()),
+ "test_size": len(task.get_testset()),
+ "task_name": dataset,
+ "signature_optimized": False,
+ "prompt_model_name": self.PROMPT_MODEL_NAME,
+ "task_model_name": self.TASK_MODEL_NAME,
+ "breadth": "NA",
+ "depth": "NA",
+ "meta_prompt_style": "default",
+ "fewshot_before": False,
+ "fewshot_after": False,
+ "temperature": self.TEMPERATURE,
+ "fewshot_candidates_num": 0,
+ "max_bootstrapped_demos": 0,
+ "bootstrapping": False,
+ "view_data": False,
+ "optimizer_log_dir": "NA",
+ "additional_notes": "",
+ "misc": "",
+ },
+ )
+
+ def test_optimizer_default(
+ self, optimizer_function, datasets=datasets, test_name="default"
+ ):
for dataset in datasets:
task = self.load_dataset(dataset)
print(f"Testing Optimizers on {dataset} ...")
dspy.settings.lm.max_tokens = task.get_max_tokens()
- evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)
- evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)
- evaluate_test = Evaluate(devset=task.get_testset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)
+ evaluate_train = Evaluate(
+ devset=task.get_trainset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
+ evaluate_dev = Evaluate(
+ devset=task.get_devset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
+ evaluate_test = Evaluate(
+ devset=task.get_testset(),
+ metric=task.get_metric(),
+ num_threads=self.NUM_THREADS,
+ display_progress=True,
+ max_errors=self.MAX_ERRORS,
+ )
default_program = task.get_program()
# Set up the optimizer kwargs
date_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
- log_dir = "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"
+ log_dir = (
+ "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"
+ )
os.makedirs(log_dir, exist_ok=True)
- kwargs = dict(breadth=self.BREADTH, depth=self.DEPTH, temperature=self.TEMPERATURE, prompt_model=self.prompt_model, view_data=False, log_dir=log_dir, metric=task.get_metric(), task_model=self.task_model)
-
+ kwargs = dict(
+ breadth=self.BREADTH,
+ depth=self.DEPTH,
+ temperature=self.TEMPERATURE,
+ prompt_model=self.prompt_model,
+ view_data=False,
+ log_dir=log_dir,
+ metric=task.get_metric(),
+ task_model=self.task_model,
+ )
+
# Optimize the default program
start = timer()
- optimized_program, output_dict = optimizer_function(default_program, task.get_trainset(), task.get_devset(), test_name, dataset, kwargs)
+ optimized_program, output_dict = optimizer_function(
+ default_program,
+ task.get_trainset(),
+ task.get_devset(),
+ test_name,
+ dataset,
+ kwargs,
+ )
end = timer()
# Evaluate the optimized program
@@ -195,37 +313,37 @@ class OptimizerTester:
print(f"Optimized dev score...")
optimized_results_dev = evaluate_dev(optimized_program)
print(f"Optimized test score...")
- optimized_results_test= evaluate_test(optimized_program)
+ optimized_results_test = evaluate_test(optimized_program)
output = {
- 'test_name': dataset + "_" + test_name,
- 'train_score': optimized_results_train,
- 'dev_score': optimized_results_dev,
- 'test_score': optimized_results_test,
- 'run_time (sec)': end - start,
- 'train_size': len(task.get_trainset()),
- 'dev_size': len(task.get_devset()),
- 'test_size': len(task.get_testset()),
- 'task_name': dataset,
- 'signature_optimized': True,
- 'prompt_model_name': self.PROMPT_MODEL_NAME,
- 'task_model_name': self.TASK_MODEL_NAME,
- 'breadth': self.BREADTH,
- 'depth': self.DEPTH,
- 'meta_prompt_style': 'default',
- 'fewshot_before': False,
- 'fewshot_after': False,
- 'temperature': self.TEMPERATURE,
- 'fewshot_candidates_num': 0,
- 'max_bootstrapped_demos': 0,
- 'bootstrapping': False,
- 'view_data': False,
- 'optimizer_log_dir': log_dir,
- 'additional_notes': '',
- 'misc': '',
+ "test_name": dataset + "_" + test_name,
+ "train_score": optimized_results_train,
+ "dev_score": optimized_results_dev,
+ "test_score": optimized_results_test,
+ "run_time (sec)": end - start,
+ "train_size": len(task.get_trainset()),
+ "dev_size": len(task.get_devset()),
+ "test_size": len(task.get_testset()),
+ "task_name": dataset,
+ "signature_optimized": True,
+ "prompt_model_name": self.PROMPT_MODEL_NAME,
+ "task_model_name": self.TASK_MODEL_NAME,
+ "breadth": self.BREADTH,
+ "depth": self.DEPTH,
+ "meta_prompt_style": "default",
+ "fewshot_before": False,
+ "fewshot_after": False,
+ "temperature": self.TEMPERATURE,
+ "fewshot_candidates_num": 0,
+ "max_bootstrapped_demos": 0,
+ "bootstrapping": False,
+ "view_data": False,
+ "optimizer_log_dir": log_dir,
+ "additional_notes": "",
+ "misc": "",
}
output.update(output_dict)
# Write the results to a csv
- self.write_to_csv('outputs', 'results.csv', output)
\ No newline at end of file
+ self.write_to_csv("outputs", "results.csv", output)
diff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.py
index 2ff5c0136..11e0c136e 100644
--- a/testing/tasks/base_task.py
+++ b/testing/tasks/base_task.py
@@ -18,7 +18,7 @@ class BaseTask(ABC):
self.TRAIN_NUM = TRAIN_NUM
return self.trainset[:TRAIN_NUM]
else:
- return self.trainset[:self.TRAIN_NUM]
+ return self.trainset[: self.TRAIN_NUM]
def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):
if TRAIN_NUM:
@@ -32,13 +32,13 @@ class BaseTask(ABC):
else:
index = max(len(self.trainset) - self.DEV_NUM, self.TRAIN_NUM)
return self.trainset[index:]
-
+
def get_testset(self, TEST_NUM=None):
if TEST_NUM:
self.TEST_NUM = TEST_NUM
return self.testset[:TEST_NUM]
else:
- return self.testset[:self.TEST_NUM]
+ return self.testset[: self.TEST_NUM]
def set_splits(self, TRAIN_NUM=None, DEV_NUM=None, TEST_NUM=None):
if TRAIN_NUM:
@@ -47,6 +47,6 @@ class BaseTask(ABC):
self.DEV_NUM = DEV_NUM
if TEST_NUM:
self.TEST_NUM = TEST_NUM
-
+
def get_max_tokens(self):
- return 150
\ No newline at end of file
+ return 150
diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.py
index fe8136b96..f57760e9d 100644
--- a/testing/tasks/gsm8k.py
+++ b/testing/tasks/gsm8k.py
@@ -13,40 +13,45 @@ class CoT(dspy.Module):
def __init__(self):
super().__init__()
self.prog = dspy.ChainOfThought("question -> answer")
-
+
def forward(self, question):
return self.prog(question=question)
-
+
+
class GSM8KTask(BaseTask):
def __init__(self):
- dataset = load_dataset("gsm8k", 'main')
+ dataset = load_dataset("gsm8k", "main")
- hf_official_train = dataset['train']
- hf_official_test = dataset['test']
+ hf_official_train = dataset["train"]
+ hf_official_test = dataset["test"]
official_train = []
official_test = []
for example in tqdm.tqdm(hf_official_train):
- question = example['question']
+ question = example["question"]
+
+ answer = example["answer"].strip().split()
+ assert answer[-2] == "####"
- answer = example['answer'].strip().split()
- assert answer[-2] == '####'
-
- gold_reasoning = ' '.join(answer[:-2])
- answer = str(int(answer[-1].replace(',', '')))
+ gold_reasoning = " ".join(answer[:-2])
+ answer = str(int(answer[-1].replace(",", "")))
- official_train.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))
+ official_train.append(
+ dict(question=question, gold_reasoning=gold_reasoning, answer=answer)
+ )
for example in tqdm.tqdm(hf_official_test):
- question = example['question']
+ question = example["question"]
- answer = example['answer'].strip().split()
- assert answer[-2] == '####'
-
- gold_reasoning = ' '.join(answer[:-2])
- answer = str(int(answer[-1].replace(',', '')))
+ answer = example["answer"].strip().split()
+ assert answer[-2] == "####"
- official_test.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))
+ gold_reasoning = " ".join(answer[:-2])
+ answer = str(int(answer[-1].replace(",", "")))
+
+ official_test.append(
+ dict(question=question, gold_reasoning=gold_reasoning, answer=answer)
+ )
rng = random.Random(0)
rng.shuffle(official_train)
@@ -58,11 +63,11 @@ class GSM8KTask(BaseTask):
devset = official_train[2000:2500]
testset = official_test[:]
- trainset = [dspy.Example(**x).with_inputs('question') for x in trainset]
- devset = [dspy.Example(**x).with_inputs('question') for x in devset]
- testset = [dspy.Example(**x).with_inputs('question') for x in testset]
+ trainset = [dspy.Example(**x).with_inputs("question") for x in trainset]
+ devset = [dspy.Example(**x).with_inputs("question") for x in devset]
+ testset = [dspy.Example(**x).with_inputs("question") for x in testset]
- self.trainset= trainset
+ self.trainset = trainset
self.devset = devset
self.testset = testset
self.metric = gsm8k_metric
@@ -71,9 +76,9 @@ class GSM8KTask(BaseTask):
def get_program(self):
return CoT()
-
+
def get_metric(self):
return self.metric
-
+
def get_max_tokens(self):
- return 600
\ No newline at end of file
+ return 600
diff --git a/testing/tasks/heart_disease.py b/testing/tasks/heart_disease.py
index 9d3638208..f64026172 100644
--- a/testing/tasks/heart_disease.py
+++ b/testing/tasks/heart_disease.py
@@ -4,27 +4,37 @@ from datasets import load_dataset
from .base_task import BaseTask
mappings = {
- 'sex': {0: 'female', 1: 'male'},
- 'cp': {1: 'typical angina', 2: 'atypical angina', 3: 'non-anginal pain', 4: 'asymptomatic'},
- 'restecg': {0: 'normal', 1: 'ST-T wave abnormality', 2: 'left ventricular hypertrophy'},
- 'exang': {0: 'no', 1: 'yes'},
- 'slope': {1: 'upsloping', 2: 'flat', 3: 'downsloping'},
- 'thal': {'3': 'normal', '6': 'fixed defect', '7': 'reversible defect'},
- 'target': {0: 'no', 1: 'yes'} # presence of heart disease
+ "sex": {0: "female", 1: "male"},
+ "cp": {
+ 1: "typical angina",
+ 2: "atypical angina",
+ 3: "non-anginal pain",
+ 4: "asymptomatic",
+ },
+ "restecg": {
+ 0: "normal",
+ 1: "ST-T wave abnormality",
+ 2: "left ventricular hypertrophy",
+ },
+ "exang": {0: "no", 1: "yes"},
+ "slope": {1: "upsloping", 2: "flat", 3: "downsloping"},
+ "thal": {"3": "normal", "6": "fixed defect", "7": "reversible defect"},
+ "target": {0: "no", 1: "yes"}, # presence of heart disease
}
dataset = load_dataset("buio/heart-disease")
fullset = []
-for x in dataset['train']:
+for x in dataset["train"]:
for key, value in x.items():
- if key in mappings: x[key] = mappings[key].get(value, value)
+ if key in mappings:
+ x[key] = mappings[key].get(value, value)
x[key] = str(x[key])
-
- inputs = [k for k in x.keys() if k != 'target']
- x['answer'] = x['target']
- del x['target']
-
+
+ inputs = [k for k in x.keys() if k != "target"]
+ x["answer"] = x["target"]
+ del x["target"]
+
fullset.append(dspy.Example(**x).with_inputs(*inputs))
random.Random(0).shuffle(fullset)
@@ -33,48 +43,101 @@ trainset = fullset[:120]
devset = trainset
testset = fullset[120:]
+
class HeartDiseaseInput(dspy.Signature):
age = dspy.InputField(desc="Age in years")
sex = dspy.InputField(desc="Sex (male or female)")
- cp = dspy.InputField(desc="Chest pain type (typical angina, atypical angina, non-anginal pain, asymptomatic)")
- trestbps = dspy.InputField(desc="Resting blood pressure (in mm Hg on admission to the hospital)")
+ cp = dspy.InputField(
+ desc="Chest pain type (typical angina, atypical angina, non-anginal pain, asymptomatic)"
+ )
+ trestbps = dspy.InputField(
+ desc="Resting blood pressure (in mm Hg on admission to the hospital)"
+ )
chol = dspy.InputField(desc="Serum cholestoral in mg/dl")
fbs = dspy.InputField(desc="Fasting blood sugar > 120 mg/dl (true or false)")
- restecg = dspy.InputField(desc="Resting electrocardiographic results (normal, ST-T wave abnormality, left ventricular hypertrophy)")
+ restecg = dspy.InputField(
+ desc="Resting electrocardiographic results (normal, ST-T wave abnormality, left ventricular hypertrophy)"
+ )
thalach = dspy.InputField(desc="Maximum heart rate achieved")
exang = dspy.InputField(desc="Exercise induced angina (yes or no)")
oldpeak = dspy.InputField(desc="ST depression induced by exercise relative to rest")
- slope = dspy.InputField(desc="The slope of the peak exercise ST segment (upsloping, flat, downsloping)")
+ slope = dspy.InputField(
+ desc="The slope of the peak exercise ST segment (upsloping, flat, downsloping)"
+ )
ca = dspy.InputField(desc="Number of major vessels (0-3) colored by flourosopy")
thal = dspy.InputField(desc="Thalassemia (normal, fixed defect, reversible defect)")
+
class HeartDiseaseSignature(HeartDiseaseInput):
"""Given patient information, predict the presence of heart disease."""
- answer = dspy.OutputField(desc="Does this patient have heart disease? Just yes or no.")
+ answer = dspy.OutputField(
+ desc="Does this patient have heart disease? Just yes or no."
+ )
+
class HeartDiseaseVote(HeartDiseaseInput):
"""Given patient information, predict the presence of heart disease. I can critically assess the provided trainee opinions."""
context = dspy.InputField(desc="A list of opinions from trainee doctors.")
- answer = dspy.OutputField(desc="Does this patient have heart disease? Just yes or no.")
+ answer = dspy.OutputField(
+ desc="Does this patient have heart disease? Just yes or no."
+ )
class Classify(dspy.Module):
def __init__(self):
- self.classify = [dspy.ChainOfThought(HeartDiseaseSignature, temperature=0.7 + i*0.01) for i in range(3)]
+ self.classify = [
+ dspy.ChainOfThought(HeartDiseaseSignature, temperature=0.7 + i * 0.01)
+ for i in range(3)
+ ]
self.vote = dspy.ChainOfThought(HeartDiseaseVote)
- def forward(self, age, sex, cp, trestbps, chol, fbs, restecg, thalach, exang, oldpeak, slope, ca, thal):
- kwargs = dict(age=age, sex=sex, cp=cp, trestbps=trestbps, chol=chol, fbs=fbs, restecg=restecg,
- thalach=thalach, exang=exang, oldpeak=oldpeak, slope=slope, ca=ca, thal=thal)
-
- opinions = [c(**kwargs) for c in self.classify]
- opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]
+ def forward(
+ self,
+ age,
+ sex,
+ cp,
+ trestbps,
+ chol,
+ fbs,
+ restecg,
+ thalach,
+ exang,
+ oldpeak,
+ slope,
+ ca,
+ thal,
+ ):
+ kwargs = dict(
+ age=age,
+ sex=sex,
+ cp=cp,
+ trestbps=trestbps,
+ chol=chol,
+ fbs=fbs,
+ restecg=restecg,
+ thalach=thalach,
+ exang=exang,
+ oldpeak=oldpeak,
+ slope=slope,
+ ca=ca,
+ thal=thal,
+ )
- opinions = [f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}." for reason, answer in opinions]
+ opinions = [c(**kwargs) for c in self.classify]
+ opinions = [
+ (opinion.reasoning.replace("\n", " ").strip("."), opinion.answer.strip("."))
+ for opinion in opinions
+ ]
+
+ opinions = [
+ f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}."
+ for reason, answer in opinions
+ ]
return self.vote(context=opinions, **kwargs)
+
class HeartDiseaseTask(BaseTask):
def __init__(self):
self.trainset = trainset
@@ -93,10 +156,10 @@ class HeartDiseaseTask(BaseTask):
def get_default_max_labeled_demos(self):
return 0
-
+
def get_trainset(self, TRAIN_NUM=None):
return self.trainset[:TRAIN_NUM]
-
+
def get_testset(self, TEST_NUM=None):
return self.testset[:TEST_NUM]
@@ -105,7 +168,6 @@ class HeartDiseaseTask(BaseTask):
def get_handwritten_prompts(self):
return []
-
+
def get_max_tokens(self):
return 350
-
\ No newline at end of file
diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.py
index 5c62b5dba..7a8966407 100644
--- a/testing/tasks/hotpotqa.py
+++ b/testing/tasks/hotpotqa.py
@@ -4,6 +4,7 @@ from dspy.evaluate import Evaluate
from .base_task import BaseTask
+
class MultiHop(dspy.Module):
def __init__(self, passages_per_hop):
super().__init__()
@@ -31,7 +32,7 @@ class HotPotQATask(BaseTask):
self.testset = [x.with_inputs("question") for x in hotpot_dataset.dev]
# Set up metrics
- NUM_THREADS=16
+ NUM_THREADS = 16
metric_EM = dspy.evaluate.answer_exact_match
self.metric = metric_EM
@@ -55,13 +56,11 @@ class HotPotQATask(BaseTask):
self.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)
def get_program(self):
- return MultiHop(
- passages_per_hop=3
- )
+ return MultiHop(passages_per_hop=3)
def get_metric(self):
return self.metric
-
+
def get_retrieval_metric(self):
return self.evaluate_retrieval
@@ -70,4 +69,3 @@ class HotPotQATask(BaseTask):
def get_default_max_labeled_demos(self):
return 2
-
diff --git a/testing/tasks/hotpotqa_conditional.py b/testing/tasks/hotpotqa_conditional.py
index c9466a834..c0b0d88fa 100644
--- a/testing/tasks/hotpotqa_conditional.py
+++ b/testing/tasks/hotpotqa_conditional.py
@@ -6,16 +6,20 @@ import os
from .base_task import BaseTask
import string
+
def preprocess_text(text):
# Remove all punctuation
- text = ''.join(char for char in text if char not in string.punctuation)
+ text = "".join(char for char in text if char not in string.punctuation)
# Convert to lower case
text = text.lower()
# Remove 'peace!' from the end if it exists
- if text.endswith('peace!'):
- text = text[:-6].strip() # Remove the last 'peace!' and strip any trailing spaces
+ if text.endswith("peace!"):
+ text = text[
+ :-6
+ ].strip() # Remove the last 'peace!' and strip any trailing spaces
return text
+
def calculate_recall(correct_words, pred_words):
correct_set = set(correct_words)
pred_set = set(pred_words)
@@ -24,7 +28,8 @@ def calculate_recall(correct_words, pred_words):
if len(correct_set) == 0:
return True # Avoid division by zero if correct answer is somehow empty
recall = len(common_words) / len(correct_set)
- return recall >= 2/3
+ return recall >= 2 / 3
+
def check_conditions(example, pred, trace=None, debug=False):
category = example.category
@@ -35,7 +40,10 @@ def check_conditions(example, pred, trace=None, debug=False):
# Check for exact match after preprocessing
if preprocessed_pred_answer != preprocessed_correct_answer:
- if debug: print(f"Exact match failed. Expected: '{preprocessed_correct_answer}', Got: '{preprocessed_pred_answer}'")
+ if debug:
+ print(
+ f"Exact match failed. Expected: '{preprocessed_correct_answer}', Got: '{preprocessed_pred_answer}'"
+ )
return False
# When the answer is a place, the response should contain no punctuation
@@ -43,7 +51,10 @@ def check_conditions(example, pred, trace=None, debug=False):
if any(char in answer for char in ",.?!;:"):
return False
else:
- if debug: print(f"Place. When the answer is a place, the response should contain no punctuation {answer}")
+ if debug:
+ print(
+ f"Place. When the answer is a place, the response should contain no punctuation {answer}"
+ )
return True
# When the answer is a date, the response should end with "Peace!"
@@ -51,7 +62,10 @@ def check_conditions(example, pred, trace=None, debug=False):
if not answer.endswith("Peace!"):
return False
else:
- if debug: print(f"Date. When the answer is a date, the response should end with Peace! {answer}")
+ if debug:
+ print(
+ f"Date. When the answer is a date, the response should end with Peace! {answer}"
+ )
return True
# When the answer is a person, the response should be entirely in lowercase
@@ -59,7 +73,10 @@ def check_conditions(example, pred, trace=None, debug=False):
if answer != answer.lower():
return False
else:
- if debug: print(f"Answer. When the answer is a person, the response should be entirely in lowercase {answer}")
+ if debug:
+ print(
+ f"Answer. When the answer is a person, the response should be entirely in lowercase {answer}"
+ )
return True
# When the answer is none of the above categories, the response should be in all caps and not end with "Peace!"
@@ -67,9 +84,13 @@ def check_conditions(example, pred, trace=None, debug=False):
if answer != answer.upper() or answer.endswith("Peace!"):
return False
else:
- if debug: print(f"Other category. the response should be in all caps and not end with Peace! {answer}")
+ if debug:
+ print(
+ f"Other category. the response should be in all caps and not end with Peace! {answer}"
+ )
return True
+
class MultiHop(dspy.Module):
def __init__(self, passages_per_hop):
super().__init__()
@@ -87,6 +108,7 @@ class MultiHop(dspy.Module):
answer=self.generate_answer(context=context, question=question).answer,
)
+
class GenerateAnswerInstruction(dspy.Signature):
"""When the answer is a person, respond entirely in lowercase. When the answer is a place, ensure your response contains no punctuation. When the answer is a date, end your response with “Peace!”. Never end your response with "Peace!" under other circumstances. When the answer is none of the above categories respond in all caps."""
@@ -94,6 +116,7 @@ class GenerateAnswerInstruction(dspy.Signature):
question = dspy.InputField(desc="Question we want an answer to")
answer = dspy.OutputField(desc="Answer to the question")
+
class MultiHopHandwritten(dspy.Module):
def __init__(self, passages_per_hop):
super().__init__()
@@ -111,17 +134,24 @@ class MultiHopHandwritten(dspy.Module):
answer=self.generate_answer(context=context, question=question).answer,
)
+
class HotPotQAConditionalTask(BaseTask):
def __init__(self):
# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqa
# Get the directory where this script is located
script_dir = os.path.dirname(os.path.abspath(__file__))
-
+
# Construct the absolute paths to the datasets
- hotpotqa_train_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_train.csv")
- hotpotqa_dev_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_dev.csv")
- hotpotqa_test_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_test.csv")
+ hotpotqa_train_path = os.path.join(
+ script_dir, "../datasets/hotpotqa_conditional/hotpot_train.csv"
+ )
+ hotpotqa_dev_path = os.path.join(
+ script_dir, "../datasets/hotpotqa_conditional/hotpot_dev.csv"
+ )
+ hotpotqa_test_path = os.path.join(
+ script_dir, "../datasets/hotpotqa_conditional/hotpot_test.csv"
+ )
# Read the datasets
hotpotqa_train = pd.read_csv(hotpotqa_train_path)
@@ -131,11 +161,25 @@ class HotPotQAConditionalTask(BaseTask):
combined_train = pd.concat([hotpotqa_train, hotpotqa_dev], ignore_index=True)
# Load and configure the datasets.
- self.trainset = [dspy.Example(question=row['question'], answer=row['answer'], category=row['answer category']).with_inputs('question') for index, row in combined_train.iterrows()]
- self.testset = [dspy.Example(question=row['question'], answer=row['answer'], category=row['answer category']).with_inputs('question') for index, row in hotpotqa_test.iterrows()]
+ self.trainset = [
+ dspy.Example(
+ question=row["question"],
+ answer=row["answer"],
+ category=row["answer category"],
+ ).with_inputs("question")
+ for index, row in combined_train.iterrows()
+ ]
+ self.testset = [
+ dspy.Example(
+ question=row["question"],
+ answer=row["answer"],
+ category=row["answer category"],
+ ).with_inputs("question")
+ for index, row in hotpotqa_test.iterrows()
+ ]
# Set up metrics
- NUM_THREADS=16
+ NUM_THREADS = 16
# TODO: set up metrics
self.metric = check_conditions
@@ -145,9 +189,7 @@ class HotPotQAConditionalTask(BaseTask):
self.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)
def get_program(self):
- return MultiHop(
- passages_per_hop=3
- )
+ return MultiHop(passages_per_hop=3)
def get_metric(self):
return self.metric
@@ -157,6 +199,6 @@ class HotPotQAConditionalTask(BaseTask):
def get_default_max_labeled_demos(self):
return 0
-
+
def get_max_tokens(self):
return 600
diff --git a/testing/tasks/hover.py b/testing/tasks/hover.py
index c7e4c1161..1a7e4378c 100644
--- a/testing/tasks/hover.py
+++ b/testing/tasks/hover.py
@@ -9,14 +9,27 @@ from dspy.evaluate import Evaluate
from .base_task import BaseTask
+
def count_unique_docs(example):
return len(set([fact["key"] for fact in example["supporting_facts"]]))
+
def discrete_retrieval_eval(example, pred, trace=None):
- gold_titles = set(map(dspy.evaluate.normalize_text, [doc['key'] for doc in example['supporting_facts']]))
- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.retrieved_docs]))
+ gold_titles = set(
+ map(
+ dspy.evaluate.normalize_text,
+ [doc["key"] for doc in example["supporting_facts"]],
+ )
+ )
+ found_titles = set(
+ map(
+ dspy.evaluate.normalize_text,
+ [c.split(" | ")[0] for c in pred.retrieved_docs],
+ )
+ )
return gold_titles.issubset(found_titles)
+
class RetrieveMultiHop(dspy.Module):
def __init__(self):
super().__init__()
@@ -27,47 +40,58 @@ class RetrieveMultiHop(dspy.Module):
self.summarize1 = dspy.ChainOfThought("claim,passages->summary")
self.summarize2 = dspy.ChainOfThought("claim,context,passages->summary")
- def forward(self,claim):
+ def forward(self, claim):
# HOP 1
hop1_docs = self.retrieve_k(claim).passages
- summary_1 = self.summarize1(claim=claim, passages=hop1_docs).summary # Summarize top k docs
-
+ summary_1 = self.summarize1(
+ claim=claim, passages=hop1_docs
+ ).summary # Summarize top k docs
+
# HOP 2
hop2_query = self.create_query_hop2(claim=claim, summary_1=summary_1).query
hop2_docs = self.retrieve_k(hop2_query).passages
- summary_2 = self.summarize2(claim=claim, context=summary_1, passages=hop2_docs).summary
+ summary_2 = self.summarize2(
+ claim=claim, context=summary_1, passages=hop2_docs
+ ).summary
# HOP 3
- hop3_query = self.create_query_hop3(claim=claim, summary_1=summary_1, summary_2=summary_2).query
+ hop3_query = self.create_query_hop3(
+ claim=claim, summary_1=summary_1, summary_2=summary_2
+ ).query
hop3_docs = self.retrieve_k(hop3_query).passages
- return dspy.Prediction(retrieved_docs = hop1_docs + hop2_docs + hop3_docs)
+ return dspy.Prediction(retrieved_docs=hop1_docs + hop2_docs + hop3_docs)
+
class HoverRetrieveDiscrete(BaseTask):
def __init__(self):
dataset = load_dataset("hover")
- hf_trainset = dataset['train']
- hf_testset = dataset['validation']
+ hf_trainset = dataset["train"]
+ hf_testset = dataset["validation"]
reformatted_hf_trainset = []
reformatted_hf_testset = []
for example in tqdm.tqdm(hf_trainset):
- claim = example['claim']
- supporting_facts = example['supporting_facts']
- label = example['label']
+ claim = example["claim"]
+ supporting_facts = example["supporting_facts"]
+ label = example["label"]
- if count_unique_docs(example) == 3: # Limit to 3 hop examples
- reformatted_hf_trainset.append(dict(claim=claim, supporting_facts=supporting_facts, label=label))
+ if count_unique_docs(example) == 3: # Limit to 3 hop examples
+ reformatted_hf_trainset.append(
+ dict(claim=claim, supporting_facts=supporting_facts, label=label)
+ )
for example in tqdm.tqdm(hf_testset):
- claim = example['claim']
- supporting_facts = example['supporting_facts']
- label = example['label']
+ claim = example["claim"]
+ supporting_facts = example["supporting_facts"]
+ label = example["label"]
if count_unique_docs(example) == 3:
- reformatted_hf_testset.append(dict(claim=claim, supporting_facts=supporting_facts, label=label))
+ reformatted_hf_testset.append(
+ dict(claim=claim, supporting_facts=supporting_facts, label=label)
+ )
rng = random.Random(0)
rng.shuffle(reformatted_hf_trainset)
@@ -77,8 +101,8 @@ class HoverRetrieveDiscrete(BaseTask):
trainset = reformatted_hf_trainset
testset = reformatted_hf_testset
- self.trainset = [dspy.Example(**x).with_inputs('claim') for x in trainset]
- self.testset = [dspy.Example(**x).with_inputs('claim') for x in testset]
+ self.trainset = [dspy.Example(**x).with_inputs("claim") for x in trainset]
+ self.testset = [dspy.Example(**x).with_inputs("claim") for x in testset]
# Set up metrics
NUM_THREADS = 16
@@ -92,16 +116,15 @@ class HoverRetrieveDiscrete(BaseTask):
def get_program(self):
return RetrieveMultiHop()
-
+
def get_metric(self):
return self.metric
-
+
def get_default_max_bootstrapped_demos(self):
return 1
def get_default_max_labeled_demos(self):
return 2
-
+
def get_max_tokens(self):
return 600
-
diff --git a/testing/tasks/iris.py b/testing/tasks/iris.py
index 64e0fcab6..ec0646f32 100644
--- a/testing/tasks/iris.py
+++ b/testing/tasks/iris.py
@@ -4,6 +4,7 @@ from .base_task import BaseTask
import random
import dspy
+
class Sig(dspy.Signature):
"Given the petal and sepal dimensions in cm, predict the iris species."
@@ -11,30 +12,51 @@ class Sig(dspy.Signature):
petal_width = dspy.InputField()
sepal_length = dspy.InputField()
sepal_width = dspy.InputField()
- answer = dspy.OutputField(desc='setosa, versicolor, or virginica')
+ answer = dspy.OutputField(desc="setosa, versicolor, or virginica")
+
class Classify(dspy.Module):
def __init__(self):
self.pred = dspy.ChainOfThought(Sig)
-
+
def forward(self, petal_length, petal_width, sepal_length, sepal_width):
- return self.pred(petal_length=petal_length, petal_width=petal_width, sepal_length=sepal_length, sepal_width=sepal_width)
-
+ return self.pred(
+ petal_length=petal_length,
+ petal_width=petal_width,
+ sepal_length=sepal_length,
+ sepal_width=sepal_width,
+ )
+
+
class IrisClassifierTask(BaseTask):
def __init__(self):
# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqa
dataset = load_dataset("hitorilabs/iris")
- fullset = [dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()}) for example in dataset['train']]
- fullset = [dspy.Example(**{**x, 'answer': ['setosa', 'versicolor', 'virginica'][int(x['species'])]}) for x in fullset]
- fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]
+ fullset = [
+ dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()})
+ for example in dataset["train"]
+ ]
+ fullset = [
+ dspy.Example(
+ **{
+ **x,
+ "answer": ["setosa", "versicolor", "virginica"][int(x["species"])],
+ }
+ )
+ for x in fullset
+ ]
+ fullset = [
+ x.with_inputs("petal_length", "petal_width", "sepal_length", "sepal_width")
+ for x in fullset
+ ]
random.Random(0).shuffle(fullset)
# self.trainset, self.devset, self.testset = fullset[:25], fullset[20:75], fullset[75:]
- self.trainset, self.testset = fullset[:75], fullset[75:]
+ self.trainset, self.testset = fullset[:75], fullset[75:]
# Set up metrics
- NUM_THREADS=16
+ NUM_THREADS = 16
# TODO: set up metrics
self.metric = dspy.evaluate.answer_exact_match
@@ -60,12 +82,12 @@ class IrisClassifierTask(BaseTask):
def get_trainset(self, TRAIN_NUM=None):
return self.trainset
-
+
def get_testset(self):
return self.testset
def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):
return self.trainset
-
+
def get_evalset(self, EVAL_NUM=None):
- return self.testset
\ No newline at end of file
+ return self.testset
diff --git a/testing/tasks/iris_typo.py b/testing/tasks/iris_typo.py
index ee0e6b3a6..9bff74ec6 100644
--- a/testing/tasks/iris_typo.py
+++ b/testing/tasks/iris_typo.py
@@ -4,6 +4,7 @@ from .base_task import BaseTask
import random
import dspy
+
class Sig(dspy.Signature):
"Given the petal and sepal dimensions in cm, predict the iris species."
# "Given the petal and sepal dimensions in cm, predict the iris species. If petal_width < 1.0, setosa. Otherwise, if petal_width < 1.65, versicolor. Otherwise, virginica."
@@ -12,29 +13,50 @@ class Sig(dspy.Signature):
petal_width = dspy.InputField()
sepal_length = dspy.InputField()
sepal_width = dspy.InputField()
- answer = dspy.OutputField(desc='setosa, versicolour, or virginica')
+ answer = dspy.OutputField(desc="setosa, versicolour, or virginica")
+
class Classify(dspy.Module):
def __init__(self):
self.pred = dspy.ChainOfThought(Sig)
-
+
def forward(self, petal_length, petal_width, sepal_length, sepal_width):
- return self.pred(petal_length=petal_length, petal_width=petal_width, sepal_length=sepal_length, sepal_width=sepal_width)
-
+ return self.pred(
+ petal_length=petal_length,
+ petal_width=petal_width,
+ sepal_length=sepal_length,
+ sepal_width=sepal_width,
+ )
+
+
class IrisTypoClassifierTask(BaseTask):
def __init__(self):
# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqa
dataset = load_dataset("hitorilabs/iris")
- fullset = [dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()}) for example in dataset['train']]
- fullset = [dspy.Example(**{**x, 'answer': ['setosa', 'versicolor', 'virginica'][int(x['species'])]}) for x in fullset]
- fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]
+ fullset = [
+ dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()})
+ for example in dataset["train"]
+ ]
+ fullset = [
+ dspy.Example(
+ **{
+ **x,
+ "answer": ["setosa", "versicolor", "virginica"][int(x["species"])],
+ }
+ )
+ for x in fullset
+ ]
+ fullset = [
+ x.with_inputs("petal_length", "petal_width", "sepal_length", "sepal_width")
+ for x in fullset
+ ]
random.Random(0).shuffle(fullset)
- self.trainset, self.testset = fullset[:75], fullset[75:]
+ self.trainset, self.testset = fullset[:75], fullset[75:]
# Set up metrics
- NUM_THREADS=16
+ NUM_THREADS = 16
# TODO: set up metrics
self.metric = dspy.evaluate.answer_exact_match
@@ -60,12 +82,12 @@ class IrisTypoClassifierTask(BaseTask):
def get_trainset(self, TRAIN_NUM=None):
return self.trainset
-
+
def get_testset(self):
return self.testset
def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):
return self.trainset
-
+
def get_evalset(self, EVAL_NUM=None):
- return self.testset
\ No newline at end of file
+ return self.testset
diff --git a/testing/tasks/scone.py b/testing/tasks/scone.py
index d0c5f5193..2c35dbf4b 100644
--- a/testing/tasks/scone.py
+++ b/testing/tasks/scone.py
@@ -13,29 +13,32 @@ def load_scone(dirname):
dfs = []
for filename in glob.glob(dirname + "/*.csv"):
df = pd.read_csv(filename, index_col=0)
- df['category'] = os.path.basename(filename).replace(".csv", "")
+ df["category"] = os.path.basename(filename).replace(".csv", "")
dfs.append(df)
data_df = pd.concat(dfs)
def as_example(row):
# The 'one_scoped' file is from an earlier dataset, MoNLI, and
# so is formatted a bit differently:
- suffix = '' if row['category'] == 'one_scoped' else '_edited'
+ suffix = "" if row["category"] == "one_scoped" else "_edited"
# Reformat the hypothesis to be an embedded clause in a question:
- hkey = 'sentence2' + suffix
- question = row[hkey][0].lower() + row[hkey][1: ].strip(".")
+ hkey = "sentence2" + suffix
+ question = row[hkey][0].lower() + row[hkey][1:].strip(".")
question = f"Can we logically conclude for sure that {question}?"
# Binary task formulation:
- label = "Yes" if row['gold_label' + suffix] == 'entailment' else "No"
- return dspy.Example({
- "context": row['sentence1' + suffix],
- "question": question,
- "answer": label,
- "category": row['category'],
- }).with_inputs("context", "question")
+ label = "Yes" if row["gold_label" + suffix] == "entailment" else "No"
+ return dspy.Example(
+ {
+ "context": row["sentence1" + suffix],
+ "question": question,
+ "answer": label,
+ "category": row["category"],
+ }
+ ).with_inputs("context", "question")
return list(data_df.apply(as_example, axis=1).values)
+
class ScoNeSignature(dspy.Signature):
("""context, question -> answer""")
@@ -43,6 +46,7 @@ class ScoNeSignature(dspy.Signature):
question = dspy.InputField()
answer = dspy.OutputField(desc="Yes or No")
+
class ScoNeCoT(dspy.Module):
def __init__(self):
super().__init__()
@@ -50,7 +54,8 @@ class ScoNeCoT(dspy.Module):
def forward(self, context, question):
return self.generate_answer(context=context, question=question)
-
+
+
class ScoNeTask(BaseTask):
def __init__(self):
# !git clone https://github.com/selenashe/ScoNe.git
@@ -64,7 +69,7 @@ class ScoNeTask(BaseTask):
random.shuffle(all_train)
# 1000 random train, 500 random dev:
- self.trainset, self.testset = all_train[: 1000], all_train[1000: 1500]
+ self.trainset, self.testset = all_train[:1000], all_train[1000:1500]
metric_EM = dspy.evaluate.answer_exact_match
self.metric = metric_EM
@@ -73,9 +78,9 @@ class ScoNeTask(BaseTask):
def get_program(self):
return ScoNeCoT()
-
+
def get_metric(self):
return self.metric
-
+
def get_max_tokens(self):
- return 200
\ No newline at end of file
+ return 200
diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.py
index 3e6c19a8c..7e366bb9b 100644
--- a/testing/tasks/tweet.py
+++ b/testing/tasks/tweet.py
@@ -17,6 +17,7 @@ class TweetSignature(dspy.Signature):
question = dspy.InputField()
answer = dspy.OutputField(desc="Yes or No")
+
class TweetCoT(dspy.Module):
def __init__(self):
super().__init__()
@@ -25,42 +26,51 @@ class TweetCoT(dspy.Module):
def forward(self, context, question):
return self.generate_answer(context=context, question=question)
+
class MultiHopTweet(dspy.Module):
- def __init__(self,passages_per_hop):
+ def __init__(self, passages_per_hop):
super().__init__()
- self.retrieve = dspy.Retrieve(k = passages_per_hop)
+ self.retrieve = dspy.Retrieve(k=passages_per_hop)
self.generate_query = dspy.ChainOfThought("context ,question->search_query")
self.generate_answer = TweetCoT()
-
- def forward (self,question) :
+
+ def forward(self, question):
context = []
for hop in range(2):
- query = self.generate_query(context = context, question = question).search_query
+ query = self.generate_query(context=context, question=question).search_query
context += self.retrieve(query).passages
- return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)
+ return dspy.Prediction(
+ context=context,
+ answer=self.generate_answer(context=context, question=question).answer,
+ )
+
# Define the signature for automatic assessments.
class Assess(dspy.Signature):
"""Assess the quality of a tweet along the specified dimension."""
- context = dspy.InputField(desc='ignore if N/A')
+ context = dspy.InputField(desc="ignore if N/A")
assessed_text = dspy.InputField()
assessment_question = dspy.InputField()
assessment_answer = dspy.OutputField(desc="Yes or No")
+
@lru_cache
def load_models():
load_dotenv() # This will load the .env file's variables
- openai.api_key = os.environ.get('OPENAI_API_KEY')
- openai.api_base = os.environ.get('OPENAI_API_BASE')
- gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')
+ openai.api_key = os.environ.get("OPENAI_API_KEY")
+ openai.api_base = os.environ.get("OPENAI_API_BASE")
+ gpt4T = dspy.OpenAI(model="gpt-3.5-turbo-1106", max_tokens=1000, model_type="chat")
retrieve = dspy.Retrieve(k=5)
return gpt4T, retrieve
+
METRIC = None
+
+
def metric(gold, pred, trace=None):
-
+
gpt4T, retrieve = load_models()
question, answer, tweet = gold.question, gold.answer, pred.answer
@@ -68,45 +78,73 @@ def metric(gold, pred, trace=None):
engaging = "Does the assessed text make for a self-contained, engaging tweet?"
faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
- correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ correct = (
+ f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ )
correct = f"{correct} Does the assessed text above contain the gold answer?"
-
- with dspy.context(lm=gpt4T):
- faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)
- correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)
- engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)
- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])
+ with dspy.context(lm=gpt4T):
+ faithful = dspy.Predict(Assess)(
+ context=context, assessed_text=tweet, assessment_question=faithful
+ )
+ correct = dspy.Predict(Assess)(
+ context="N/A", assessed_text=tweet, assessment_question=correct
+ )
+ engaging = dspy.Predict(Assess)(
+ context="N/A", assessed_text=tweet, assessment_question=engaging
+ )
+
+ correct, engaging, faithful = (
+ m.assessment_answer.split()[0].lower() == "yes"
+ for m in [correct, engaging, faithful]
+ )
score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
if METRIC is not None:
- if METRIC == 'correct': return correct
- if METRIC == 'engaging': return engaging
- if METRIC == 'faithful': return faithful
-
- if trace is not None: return score >= 3
+ if METRIC == "correct":
+ return correct
+ if METRIC == "engaging":
+ return engaging
+ if METRIC == "faithful":
+ return faithful
+
+ if trace is not None:
+ return score >= 3
return score / 3.0
+
class TweetTask(BaseTask):
def __init__(self):
# Load the dataset.
- dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)
+ dataset = HotPotQA(
+ train_seed=1,
+ train_size=500,
+ eval_seed=2023,
+ dev_size=200,
+ test_size=0,
+ keep_details=True,
+ )
# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
- self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
- self.testset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
+ self.trainset = [
+ x.without("id", "type").with_inputs("question") for x in dataset.train
+ ]
+ self.testset = [
+ x.without("id", "type").with_inputs("question") for x in dataset.dev
+ ]
- self.trainset = [x.with_inputs('question') for x in dataset.train]
- self.testset = [x.with_inputs('question') for x in dataset.dev]
+ self.trainset = [x.with_inputs("question") for x in dataset.train]
+ self.testset = [x.with_inputs("question") for x in dataset.dev]
self.metric = metric
self.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)
def get_program(self):
- return MultiHopTweet(passages_per_hop=3) #TODO: make it so we can specify # of passages
-
+ return MultiHopTweet(
+ passages_per_hop=3
+ ) # TODO: make it so we can specify # of passages
+
def get_metric(self):
return self.metric
-
diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.py
index a1c6f0464..5eec5f057 100644
--- a/testing/tasks/tweet_metric.py
+++ b/testing/tasks/tweet_metric.py
@@ -20,6 +20,7 @@ class TweetSignature(dspy.Signature):
question = dspy.InputField()
answer = dspy.OutputField(desc="Yes or No")
+
class TweetCoT(dspy.Module):
def __init__(self):
super().__init__()
@@ -28,61 +29,89 @@ class TweetCoT(dspy.Module):
def forward(self, context, question):
return self.generate_answer(context=context, question=question)
+
class MultiHopTweet(dspy.Module):
- def __init__(self,passages_per_hop):
+ def __init__(self, passages_per_hop):
super().__init__()
- self.retrieve = dspy.Retrieve(k = passages_per_hop)
+ self.retrieve = dspy.Retrieve(k=passages_per_hop)
self.generate_query = dspy.ChainOfThought("context ,question->search_query")
self.generate_answer = TweetCoT()
-
- def forward (self,question) :
+
+ def forward(self, question):
context = []
for hop in range(2):
- query = self.generate_query(context = context, question = question).search_query
+ query = self.generate_query(context=context, question=question).search_query
context += self.retrieve(query).passages
- return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)
+ return dspy.Prediction(
+ context=context,
+ answer=self.generate_answer(context=context, question=question).answer,
+ )
+
# Define the signature for automatic assessments.
class Assess(dspy.Signature):
"""Assess the quality of a tweet along the specified dimension."""
- context = dspy.InputField(desc='ignore if N/A')
+ context = dspy.InputField(desc="ignore if N/A")
assessed_text = dspy.InputField()
assessment_question = dspy.InputField()
assessment_answer = dspy.OutputField(desc="Yes or No")
+
@lru_cache
def load_models():
load_dotenv() # This will load the .env file's variables
- openai.api_key = os.environ.get('OPENAI_API_KEY')
- openai.api_base = os.environ.get('OPENAI_API_BASE')
- gpt3T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')
- gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')
+ openai.api_key = os.environ.get("OPENAI_API_KEY")
+ openai.api_base = os.environ.get("OPENAI_API_BASE")
+ gpt3T = dspy.OpenAI(model="gpt-3.5-turbo-1106", max_tokens=1000, model_type="chat")
+ gpt4T = dspy.OpenAI(model="gpt-4-1106-preview", max_tokens=1000, model_type="chat")
retrieve = dspy.Retrieve(k=5)
return gpt3T, gpt4T, retrieve
+
METRIC = None
+
+
def metric(gold, pred, trace=None):
gpt3T, gpt4T, retrieve = load_models()
- question, answer, tweet, context = gold.question, gold.answer, gold.tweet, gold.context
+ question, answer, tweet, context = (
+ gold.question,
+ gold.answer,
+ gold.tweet,
+ gold.context,
+ )
score_pred = pred.score
engaging = "Does the assessed text make for a self-contained, engaging tweet?"
faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
- correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ correct = (
+ f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ )
correct = f"{correct} Does the assessed text above contain the gold answer?"
-
- with dspy.context(lm=gpt3T): # TODO Update to GPT4
- faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)
- correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)
- engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)
- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])
+ with dspy.context(lm=gpt3T): # TODO Update to GPT4
+ faithful = dspy.Predict(Assess)(
+ context=context, assessed_text=tweet, assessment_question=faithful
+ )
+ correct = dspy.Predict(Assess)(
+ context="N/A", assessed_text=tweet, assessment_question=correct
+ )
+ engaging = dspy.Predict(Assess)(
+ context="N/A", assessed_text=tweet, assessment_question=engaging
+ )
+
+ correct, engaging, faithful = (
+ m.assessment_answer.split()[0].lower() == "yes"
+ for m in [correct, engaging, faithful]
+ )
score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
- return 1 - abs(score - score_pred) # We want a score we can maximize, so take the negative L1 norm and add 1
+ return 1 - abs(
+ score - score_pred
+ ) # We want a score we can maximize, so take the negative L1 norm and add 1
+
class TweetMetric(dspy.Module):
def __init__(self):
@@ -90,31 +119,54 @@ class TweetMetric(dspy.Module):
self.engaging = dspy.Predict(Assess)
self.faithful = dspy.Predict(Assess)
self.correct = dspy.Predict(Assess)
-
- def forward (self, tweet, context, question, answer) :
+
+ def forward(self, tweet, context, question, answer):
engaging = "Does the assessed text make for a self-contained, engaging tweet?"
faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
correct = f"{correct} Does the assessed text above contain the gold answer?"
-
- faithful = self.faithful(context=context, assessed_text=tweet, assessment_question=faithful)
- correct = self.correct(context='N/A', assessed_text=tweet, assessment_question=correct)
- engaging = self.engaging(context='N/A', assessed_text=tweet, assessment_question=engaging)
- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])
- score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
+ faithful = self.faithful(
+ context=context, assessed_text=tweet, assessment_question=faithful
+ )
+ correct = self.correct(
+ context="N/A", assessed_text=tweet, assessment_question=correct
+ )
+ engaging = self.engaging(
+ context="N/A", assessed_text=tweet, assessment_question=engaging
+ )
+
+ correct, engaging, faithful = (
+ m.assessment_answer.split()[0].lower() == "yes"
+ for m in [correct, engaging, faithful]
+ )
+ score = (
+ (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
+ )
+
+ return dspy.Prediction(score=score / 3.0)
- return dspy.Prediction(score= score/3.0)
class TweetMetricTask(BaseTask):
def __init__(self):
# Load the dataset.
- dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)
+ dataset = HotPotQA(
+ train_seed=1,
+ train_size=500,
+ eval_seed=2023,
+ dev_size=200,
+ test_size=0,
+ keep_details=True,
+ )
# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
- trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
- devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
+ trainset_temp = [
+ x.without("id", "type").with_inputs("question") for x in dataset.train
+ ]
+ devset_temp = [
+ x.without("id", "type").with_inputs("question") for x in dataset.dev
+ ]
self.trainset = []
self.testset = []
@@ -126,18 +178,36 @@ class TweetMetricTask(BaseTask):
question = ex.question
answer = ex.answer
tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer
- example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}
-
- self.trainset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='train').with_inputs('context', 'question', 'answer', 'tweet'))
+ example = {
+ "context": context,
+ "question": question,
+ "answer": answer,
+ "tweet": tweet,
+ }
+
+ self.trainset.append(
+ Example(
+ **example, dspy_uuid=str(uuid.uuid4()), dspy_split="train"
+ ).with_inputs("context", "question", "answer", "tweet")
+ )
for ex in tqdm(devset_temp, desc="Preprocessing Devset"):
context = retrieve(ex.question).passages
question = ex.question
answer = ex.answer
tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer
- example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}
-
- self.testset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))
+ example = {
+ "context": context,
+ "question": question,
+ "answer": answer,
+ "tweet": tweet,
+ }
+
+ self.testset.append(
+ Example(
+ **example, dspy_uuid=str(uuid.uuid4()), dspy_split="dev"
+ ).with_inputs("context", "question", "answer", "tweet")
+ )
self.metric = metric
@@ -145,7 +215,6 @@ class TweetMetricTask(BaseTask):
def get_program(self):
return TweetMetric()
-
+
def get_metric(self):
return self.metric
-