DSPy: The framework for programming—not prompting—language models
Python
formatting updates
klopsahlongc223c3be48ed
diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.pyindex 09cae6b62..841512472 100644--- a/testing/optimizer_tester.py+++ b/testing/optimizer_tester.py@@ -20,14 +20,38 @@ from tasks.hover import HoverRetrieveDiscretefrom tasks.iris_typo import IrisTypoClassifierTaskfrom tasks.iris import IrisClassifierTask-datasets = ["scone", "hotpotqa", "hotpotqa_conditional", "gsm8k", "tweet", "heart_disease", "iris", "iris_typo", "hover_retrieve_discrete", "tweet_metric"]+datasets = [+ "scone",+ "hotpotqa",+ "hotpotqa_conditional",+ "gsm8k",+ "tweet",+ "heart_disease",+ "iris",+ "iris_typo",+ "hover_retrieve_discrete",+ "tweet_metric",+]+class OptimizerTester:- def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,- num_threads = 10, default_breadth = 10, default_depth = 3, default_temperature = 1.1,- prompt_model_name = "gpt-3.5-turbo-1106", task_model_name = "meta-llama/Llama-2-13b-chat-hf",- prompt_model = None, task_model = None, max_errors=100,- colbert_v2_endpoint = "http://20.102.90.50:2017/wiki17_abstracts"):+ def __init__(+ self,+ datasets=datasets,+ default_train_num=200,+ default_dev_num=100,+ default_test_num=200,+ num_threads=10,+ default_breadth=10,+ default_depth=3,+ default_temperature=1.1,+ prompt_model_name="gpt-3.5-turbo-1106",+ task_model_name="meta-llama/Llama-2-13b-chat-hf",+ prompt_model=None,+ task_model=None,+ max_errors=100,+ colbert_v2_endpoint="http://20.102.90.50:2017/wiki17_abstracts",+ ):self.datasets = datasetsself.TRAIN_NUM = default_train_numself.DEV_NUM = default_dev_num@@ -43,18 +67,24 @@ class OptimizerTester:load_dotenv() # This will load the .env file's variables- openai.api_key = os.environ.get('OPENAI_API_KEY')- openai.api_base = os.environ.get('OPENAI_API_BASE')+ openai.api_key = os.environ.get("OPENAI_API_KEY")+ openai.api_base = os.environ.get("OPENAI_API_BASE")# Prompt gen modelif not prompt_model:- self.prompt_model = dspy.OpenAI(model=self.PROMPT_MODEL_NAME, max_tokens=700)+ self.prompt_model = dspy.OpenAI(+ model=self.PROMPT_MODEL_NAME, max_tokens=700+ )else:self.prompt_model = prompt_model# Task modelif not task_model:- self.task_model = dspy.HFClientTGI(model=self.TASK_MODEL_NAME, port=[7140, 7141, 7142, 7143], max_tokens=150)+ self.task_model = dspy.HFClientTGI(+ model=self.TASK_MODEL_NAME,+ port=[7140, 7141, 7142, 7143],+ max_tokens=150,+ )else:self.task_model = task_modelself.colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)@@ -65,21 +95,47 @@ class OptimizerTester:# Ensure the output directory existsos.makedirs(folder_name, exist_ok=True)file_path = os.path.join(folder_name, file_name)-+# Check if file exists to determine if headers should be writtenfile_exists = os.path.isfile(file_path)- with open(file_path, mode='a', newline='', encoding='utf-8') as file:+ with open(file_path, mode="a", newline="", encoding="utf-8") as file:writer = csv.writer(file)- headers = ['test_name', 'train_score', 'dev_score', 'test_score', 'run_time (sec)', 'train_size', 'dev_size', 'test_size', 'task_name', 'signature_optimized', 'prompt_model_name', 'task_model_name', 'breadth', 'depth', 'meta_prompt_style', 'fewshot_before', 'fewshot_after', 'temperature', 'fewshot_candidates_num', 'max_bootstrapped_demos', 'bootstrapping', 'view_data', 'optimizer_log_dir', 'additional_notes', 'misc']-+ headers = [+ "test_name",+ "train_score",+ "dev_score",+ "test_score",+ "run_time (sec)",+ "train_size",+ "dev_size",+ "test_size",+ "task_name",+ "signature_optimized",+ "prompt_model_name",+ "task_model_name",+ "breadth",+ "depth",+ "meta_prompt_style",+ "fewshot_before",+ "fewshot_after",+ "temperature",+ "fewshot_candidates_num",+ "max_bootstrapped_demos",+ "bootstrapping",+ "view_data",+ "optimizer_log_dir",+ "additional_notes",+ "misc",+ ]+# Write headers if the file is being createdif not file_exists:writer.writerow(headers)-+# Format the data- formatted_data = ['NA'] * len(headers)+ formatted_data = ["NA"] * len(headers)formatted_data[-1] = ""for key in data.keys():if key in headers:@@ -89,7 +145,7 @@ class OptimizerTester:# Write the datawriter.writerow(formatted_data)- def load_dataset(self,dataset):+ def load_dataset(self, dataset):ds = Nonedataset = dataset.lower()if dataset == "scone":@@ -114,9 +170,11 @@ class OptimizerTester:ds = TweetMetricTask()else:raise ValueError("Invalid dataset name.")- ds.set_splits(TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, TEST_NUM=self.TEST_NUM)+ ds.set_splits(+ TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, TEST_NUM=self.TEST_NUM+ )return ds-+# Computes baseline results for a given datasetdef test_baseline(self, datasets=datasets, test_name="baseline"):for dataset in datasets:@@ -124,9 +182,27 @@ class OptimizerTester:task = self.load_dataset(dataset)dspy.settings.lm.max_tokens = task.get_max_tokens()- evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)- evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)- evaluate_test = Evaluate(devset=task.get_testset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors=self.MAX_ERRORS)+ evaluate_train = Evaluate(+ devset=task.get_trainset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )+ evaluate_dev = Evaluate(+ devset=task.get_devset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )+ evaluate_test = Evaluate(+ devset=task.get_testset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )default_program = task.get_program()# Evaluate the default program@@ -138,55 +214,97 @@ class OptimizerTester:default_results_test = evaluate_test(default_program)# Write the results to a csv- self.write_to_csv('outputs', 'results.csv', {- 'test_name': dataset + "_" + test_name,- 'train_score': default_results_train,- 'dev_score': default_results_dev,- 'test_score': default_results_test,- 'run_time (sec)': 0,- 'train_size': len(task.get_trainset()),- 'dev_size': len(task.get_devset()),- 'test_size': len(task.get_testset()),- 'task_name': dataset,- 'signature_optimized': False,- 'prompt_model_name': self.PROMPT_MODEL_NAME,- 'task_model_name': self.TASK_MODEL_NAME,- 'breadth': 'NA',- 'depth': 'NA',- 'meta_prompt_style': 'default',- 'fewshot_before': False,- 'fewshot_after': False,- 'temperature': self.TEMPERATURE,- 'fewshot_candidates_num': 0,- 'max_bootstrapped_demos': 0,- 'bootstrapping': False,- 'view_data': False,- 'optimizer_log_dir': 'NA',- 'additional_notes': '',- 'misc': '',- })-- def test_optimizer_default(self, optimizer_function, datasets=datasets, test_name="default"):+ self.write_to_csv(+ "outputs",+ "results.csv",+ {+ "test_name": dataset + "_" + test_name,+ "train_score": default_results_train,+ "dev_score": default_results_dev,+ "test_score": default_results_test,+ "run_time (sec)": 0,+ "train_size": len(task.get_trainset()),+ "dev_size": len(task.get_devset()),+ "test_size": len(task.get_testset()),+ "task_name": dataset,+ "signature_optimized": False,+ "prompt_model_name": self.PROMPT_MODEL_NAME,+ "task_model_name": self.TASK_MODEL_NAME,+ "breadth": "NA",+ "depth": "NA",+ "meta_prompt_style": "default",+ "fewshot_before": False,+ "fewshot_after": False,+ "temperature": self.TEMPERATURE,+ "fewshot_candidates_num": 0,+ "max_bootstrapped_demos": 0,+ "bootstrapping": False,+ "view_data": False,+ "optimizer_log_dir": "NA",+ "additional_notes": "",+ "misc": "",+ },+ )++ def test_optimizer_default(+ self, optimizer_function, datasets=datasets, test_name="default"+ ):for dataset in datasets:task = self.load_dataset(dataset)print(f"Testing Optimizers on {dataset} ...")dspy.settings.lm.max_tokens = task.get_max_tokens()- evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)- evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)- evaluate_test = Evaluate(devset=task.get_testset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True, max_errors = self.MAX_ERRORS)+ evaluate_train = Evaluate(+ devset=task.get_trainset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )+ evaluate_dev = Evaluate(+ devset=task.get_devset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )+ evaluate_test = Evaluate(+ devset=task.get_testset(),+ metric=task.get_metric(),+ num_threads=self.NUM_THREADS,+ display_progress=True,+ max_errors=self.MAX_ERRORS,+ )default_program = task.get_program()# Set up the optimizer kwargsdate_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")- log_dir = "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"+ log_dir = (+ "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"+ )os.makedirs(log_dir, exist_ok=True)- kwargs = dict(breadth=self.BREADTH, depth=self.DEPTH, temperature=self.TEMPERATURE, prompt_model=self.prompt_model, view_data=False, log_dir=log_dir, metric=task.get_metric(), task_model=self.task_model)-+ kwargs = dict(+ breadth=self.BREADTH,+ depth=self.DEPTH,+ temperature=self.TEMPERATURE,+ prompt_model=self.prompt_model,+ view_data=False,+ log_dir=log_dir,+ metric=task.get_metric(),+ task_model=self.task_model,+ )+# Optimize the default programstart = timer()- optimized_program, output_dict = optimizer_function(default_program, task.get_trainset(), task.get_devset(), test_name, dataset, kwargs)+ optimized_program, output_dict = optimizer_function(+ default_program,+ task.get_trainset(),+ task.get_devset(),+ test_name,+ dataset,+ kwargs,+ )end = timer()# Evaluate the optimized program@@ -195,37 +313,37 @@ class OptimizerTester:print(f"Optimized dev score...")optimized_results_dev = evaluate_dev(optimized_program)print(f"Optimized test score...")- optimized_results_test= evaluate_test(optimized_program)+ optimized_results_test = evaluate_test(optimized_program)output = {- 'test_name': dataset + "_" + test_name,- 'train_score': optimized_results_train,- 'dev_score': optimized_results_dev,- 'test_score': optimized_results_test,- 'run_time (sec)': end - start,- 'train_size': len(task.get_trainset()),- 'dev_size': len(task.get_devset()),- 'test_size': len(task.get_testset()),- 'task_name': dataset,- 'signature_optimized': True,- 'prompt_model_name': self.PROMPT_MODEL_NAME,- 'task_model_name': self.TASK_MODEL_NAME,- 'breadth': self.BREADTH,- 'depth': self.DEPTH,- 'meta_prompt_style': 'default',- 'fewshot_before': False,- 'fewshot_after': False,- 'temperature': self.TEMPERATURE,- 'fewshot_candidates_num': 0,- 'max_bootstrapped_demos': 0,- 'bootstrapping': False,- 'view_data': False,- 'optimizer_log_dir': log_dir,- 'additional_notes': '',- 'misc': '',+ "test_name": dataset + "_" + test_name,+ "train_score": optimized_results_train,+ "dev_score": optimized_results_dev,+ "test_score": optimized_results_test,+ "run_time (sec)": end - start,+ "train_size": len(task.get_trainset()),+ "dev_size": len(task.get_devset()),+ "test_size": len(task.get_testset()),+ "task_name": dataset,+ "signature_optimized": True,+ "prompt_model_name": self.PROMPT_MODEL_NAME,+ "task_model_name": self.TASK_MODEL_NAME,+ "breadth": self.BREADTH,+ "depth": self.DEPTH,+ "meta_prompt_style": "default",+ "fewshot_before": False,+ "fewshot_after": False,+ "temperature": self.TEMPERATURE,+ "fewshot_candidates_num": 0,+ "max_bootstrapped_demos": 0,+ "bootstrapping": False,+ "view_data": False,+ "optimizer_log_dir": log_dir,+ "additional_notes": "",+ "misc": "",}output.update(output_dict)# Write the results to a csv- self.write_to_csv('outputs', 'results.csv', output)\ No newline at end of file+ self.write_to_csv("outputs", "results.csv", output)diff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.pyindex 2ff5c0136..11e0c136e 100644--- a/testing/tasks/base_task.py+++ b/testing/tasks/base_task.py@@ -18,7 +18,7 @@ class BaseTask(ABC):self.TRAIN_NUM = TRAIN_NUMreturn self.trainset[:TRAIN_NUM]else:- return self.trainset[:self.TRAIN_NUM]+ return self.trainset[: self.TRAIN_NUM]def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):if TRAIN_NUM:@@ -32,13 +32,13 @@ class BaseTask(ABC):else:index = max(len(self.trainset) - self.DEV_NUM, self.TRAIN_NUM)return self.trainset[index:]-+def get_testset(self, TEST_NUM=None):if TEST_NUM:self.TEST_NUM = TEST_NUMreturn self.testset[:TEST_NUM]else:- return self.testset[:self.TEST_NUM]+ return self.testset[: self.TEST_NUM]def set_splits(self, TRAIN_NUM=None, DEV_NUM=None, TEST_NUM=None):if TRAIN_NUM:@@ -47,6 +47,6 @@ class BaseTask(ABC):self.DEV_NUM = DEV_NUMif TEST_NUM:self.TEST_NUM = TEST_NUM-+def get_max_tokens(self):- return 150\ No newline at end of file+ return 150diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.pyindex fe8136b96..f57760e9d 100644--- a/testing/tasks/gsm8k.py+++ b/testing/tasks/gsm8k.py@@ -13,40 +13,45 @@ class CoT(dspy.Module):def __init__(self):super().__init__()self.prog = dspy.ChainOfThought("question -> answer")-+def forward(self, question):return self.prog(question=question)-++class GSM8KTask(BaseTask):def __init__(self):- dataset = load_dataset("gsm8k", 'main')+ dataset = load_dataset("gsm8k", "main")- hf_official_train = dataset['train']- hf_official_test = dataset['test']+ hf_official_train = dataset["train"]+ hf_official_test = dataset["test"]official_train = []official_test = []for example in tqdm.tqdm(hf_official_train):- question = example['question']+ question = example["question"]++ answer = example["answer"].strip().split()+ assert answer[-2] == "####"- answer = example['answer'].strip().split()- assert answer[-2] == '####'-- gold_reasoning = ' '.join(answer[:-2])- answer = str(int(answer[-1].replace(',', '')))+ gold_reasoning = " ".join(answer[:-2])+ answer = str(int(answer[-1].replace(",", "")))- official_train.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))+ official_train.append(+ dict(question=question, gold_reasoning=gold_reasoning, answer=answer)+ )for example in tqdm.tqdm(hf_official_test):- question = example['question']+ question = example["question"]- answer = example['answer'].strip().split()- assert answer[-2] == '####'-- gold_reasoning = ' '.join(answer[:-2])- answer = str(int(answer[-1].replace(',', '')))+ answer = example["answer"].strip().split()+ assert answer[-2] == "####"- official_test.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))+ gold_reasoning = " ".join(answer[:-2])+ answer = str(int(answer[-1].replace(",", "")))++ official_test.append(+ dict(question=question, gold_reasoning=gold_reasoning, answer=answer)+ )rng = random.Random(0)rng.shuffle(official_train)@@ -58,11 +63,11 @@ class GSM8KTask(BaseTask):devset = official_train[2000:2500]testset = official_test[:]- trainset = [dspy.Example(**x).with_inputs('question') for x in trainset]- devset = [dspy.Example(**x).with_inputs('question') for x in devset]- testset = [dspy.Example(**x).with_inputs('question') for x in testset]+ trainset = [dspy.Example(**x).with_inputs("question") for x in trainset]+ devset = [dspy.Example(**x).with_inputs("question") for x in devset]+ testset = [dspy.Example(**x).with_inputs("question") for x in testset]- self.trainset= trainset+ self.trainset = trainsetself.devset = devsetself.testset = testsetself.metric = gsm8k_metric@@ -71,9 +76,9 @@ class GSM8KTask(BaseTask):def get_program(self):return CoT()-+def get_metric(self):return self.metric-+def get_max_tokens(self):- return 600\ No newline at end of file+ return 600diff --git a/testing/tasks/heart_disease.py b/testing/tasks/heart_disease.pyindex 9d3638208..f64026172 100644--- a/testing/tasks/heart_disease.py+++ b/testing/tasks/heart_disease.py@@ -4,27 +4,37 @@ from datasets import load_datasetfrom .base_task import BaseTaskmappings = {- 'sex': {0: 'female', 1: 'male'},- 'cp': {1: 'typical angina', 2: 'atypical angina', 3: 'non-anginal pain', 4: 'asymptomatic'},- 'restecg': {0: 'normal', 1: 'ST-T wave abnormality', 2: 'left ventricular hypertrophy'},- 'exang': {0: 'no', 1: 'yes'},- 'slope': {1: 'upsloping', 2: 'flat', 3: 'downsloping'},- 'thal': {'3': 'normal', '6': 'fixed defect', '7': 'reversible defect'},- 'target': {0: 'no', 1: 'yes'} # presence of heart disease+ "sex": {0: "female", 1: "male"},+ "cp": {+ 1: "typical angina",+ 2: "atypical angina",+ 3: "non-anginal pain",+ 4: "asymptomatic",+ },+ "restecg": {+ 0: "normal",+ 1: "ST-T wave abnormality",+ 2: "left ventricular hypertrophy",+ },+ "exang": {0: "no", 1: "yes"},+ "slope": {1: "upsloping", 2: "flat", 3: "downsloping"},+ "thal": {"3": "normal", "6": "fixed defect", "7": "reversible defect"},+ "target": {0: "no", 1: "yes"}, # presence of heart disease}dataset = load_dataset("buio/heart-disease")fullset = []-for x in dataset['train']:+for x in dataset["train"]:for key, value in x.items():- if key in mappings: x[key] = mappings[key].get(value, value)+ if key in mappings:+ x[key] = mappings[key].get(value, value)x[key] = str(x[key])-- inputs = [k for k in x.keys() if k != 'target']- x['answer'] = x['target']- del x['target']-++ inputs = [k for k in x.keys() if k != "target"]+ x["answer"] = x["target"]+ del x["target"]+fullset.append(dspy.Example(**x).with_inputs(*inputs))random.Random(0).shuffle(fullset)@@ -33,48 +43,101 @@ trainset = fullset[:120]devset = trainsettestset = fullset[120:]+class HeartDiseaseInput(dspy.Signature):age = dspy.InputField(desc="Age in years")sex = dspy.InputField(desc="Sex (male or female)")- cp = dspy.InputField(desc="Chest pain type (typical angina, atypical angina, non-anginal pain, asymptomatic)")- trestbps = dspy.InputField(desc="Resting blood pressure (in mm Hg on admission to the hospital)")+ cp = dspy.InputField(+ desc="Chest pain type (typical angina, atypical angina, non-anginal pain, asymptomatic)"+ )+ trestbps = dspy.InputField(+ desc="Resting blood pressure (in mm Hg on admission to the hospital)"+ )chol = dspy.InputField(desc="Serum cholestoral in mg/dl")fbs = dspy.InputField(desc="Fasting blood sugar > 120 mg/dl (true or false)")- restecg = dspy.InputField(desc="Resting electrocardiographic results (normal, ST-T wave abnormality, left ventricular hypertrophy)")+ restecg = dspy.InputField(+ desc="Resting electrocardiographic results (normal, ST-T wave abnormality, left ventricular hypertrophy)"+ )thalach = dspy.InputField(desc="Maximum heart rate achieved")exang = dspy.InputField(desc="Exercise induced angina (yes or no)")oldpeak = dspy.InputField(desc="ST depression induced by exercise relative to rest")- slope = dspy.InputField(desc="The slope of the peak exercise ST segment (upsloping, flat, downsloping)")+ slope = dspy.InputField(+ desc="The slope of the peak exercise ST segment (upsloping, flat, downsloping)"+ )ca = dspy.InputField(desc="Number of major vessels (0-3) colored by flourosopy")thal = dspy.InputField(desc="Thalassemia (normal, fixed defect, reversible defect)")+class HeartDiseaseSignature(HeartDiseaseInput):"""Given patient information, predict the presence of heart disease."""- answer = dspy.OutputField(desc="Does this patient have heart disease? Just yes or no.")+ answer = dspy.OutputField(+ desc="Does this patient have heart disease? Just yes or no."+ )+class HeartDiseaseVote(HeartDiseaseInput):"""Given patient information, predict the presence of heart disease. I can critically assess the provided trainee opinions."""context = dspy.InputField(desc="A list of opinions from trainee doctors.")- answer = dspy.OutputField(desc="Does this patient have heart disease? Just yes or no.")+ answer = dspy.OutputField(+ desc="Does this patient have heart disease? Just yes or no."+ )class Classify(dspy.Module):def __init__(self):- self.classify = [dspy.ChainOfThought(HeartDiseaseSignature, temperature=0.7 + i*0.01) for i in range(3)]+ self.classify = [+ dspy.ChainOfThought(HeartDiseaseSignature, temperature=0.7 + i * 0.01)+ for i in range(3)+ ]self.vote = dspy.ChainOfThought(HeartDiseaseVote)- def forward(self, age, sex, cp, trestbps, chol, fbs, restecg, thalach, exang, oldpeak, slope, ca, thal):- kwargs = dict(age=age, sex=sex, cp=cp, trestbps=trestbps, chol=chol, fbs=fbs, restecg=restecg,- thalach=thalach, exang=exang, oldpeak=oldpeak, slope=slope, ca=ca, thal=thal)-- opinions = [c(**kwargs) for c in self.classify]- opinions = [(opinion.reasoning.replace('\n', ' ').strip('.'), opinion.answer.strip('.')) for opinion in opinions]+ def forward(+ self,+ age,+ sex,+ cp,+ trestbps,+ chol,+ fbs,+ restecg,+ thalach,+ exang,+ oldpeak,+ slope,+ ca,+ thal,+ ):+ kwargs = dict(+ age=age,+ sex=sex,+ cp=cp,+ trestbps=trestbps,+ chol=chol,+ fbs=fbs,+ restecg=restecg,+ thalach=thalach,+ exang=exang,+ oldpeak=oldpeak,+ slope=slope,+ ca=ca,+ thal=thal,+ )- opinions = [f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}." for reason, answer in opinions]+ opinions = [c(**kwargs) for c in self.classify]+ opinions = [+ (opinion.reasoning.replace("\n", " ").strip("."), opinion.answer.strip("."))+ for opinion in opinions+ ]++ opinions = [+ f"I'm a trainee doctor, trying to {reason}. Hence, my answer is {answer}."+ for reason, answer in opinions+ ]return self.vote(context=opinions, **kwargs)+class HeartDiseaseTask(BaseTask):def __init__(self):self.trainset = trainset@@ -93,10 +156,10 @@ class HeartDiseaseTask(BaseTask):def get_default_max_labeled_demos(self):return 0-+def get_trainset(self, TRAIN_NUM=None):return self.trainset[:TRAIN_NUM]-+def get_testset(self, TEST_NUM=None):return self.testset[:TEST_NUM]@@ -105,7 +168,6 @@ class HeartDiseaseTask(BaseTask):def get_handwritten_prompts(self):return []-+def get_max_tokens(self):return 350-\ No newline at end of filediff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.pyindex 5c62b5dba..7a8966407 100644--- a/testing/tasks/hotpotqa.py+++ b/testing/tasks/hotpotqa.py@@ -4,6 +4,7 @@ from dspy.evaluate import Evaluatefrom .base_task import BaseTask+class MultiHop(dspy.Module):def __init__(self, passages_per_hop):super().__init__()@@ -31,7 +32,7 @@ class HotPotQATask(BaseTask):self.testset = [x.with_inputs("question") for x in hotpot_dataset.dev]# Set up metrics- NUM_THREADS=16+ NUM_THREADS = 16metric_EM = dspy.evaluate.answer_exact_matchself.metric = metric_EM@@ -55,13 +56,11 @@ class HotPotQATask(BaseTask):self.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)def get_program(self):- return MultiHop(- passages_per_hop=3- )+ return MultiHop(passages_per_hop=3)def get_metric(self):return self.metric-+def get_retrieval_metric(self):return self.evaluate_retrieval@@ -70,4 +69,3 @@ class HotPotQATask(BaseTask):def get_default_max_labeled_demos(self):return 2-diff --git a/testing/tasks/hotpotqa_conditional.py b/testing/tasks/hotpotqa_conditional.pyindex c9466a834..c0b0d88fa 100644--- a/testing/tasks/hotpotqa_conditional.py+++ b/testing/tasks/hotpotqa_conditional.py@@ -6,16 +6,20 @@ import osfrom .base_task import BaseTaskimport string+def preprocess_text(text):# Remove all punctuation- text = ''.join(char for char in text if char not in string.punctuation)+ text = "".join(char for char in text if char not in string.punctuation)# Convert to lower casetext = text.lower()# Remove 'peace!' from the end if it exists- if text.endswith('peace!'):- text = text[:-6].strip() # Remove the last 'peace!' and strip any trailing spaces+ if text.endswith("peace!"):+ text = text[+ :-6+ ].strip() # Remove the last 'peace!' and strip any trailing spacesreturn text+def calculate_recall(correct_words, pred_words):correct_set = set(correct_words)pred_set = set(pred_words)@@ -24,7 +28,8 @@ def calculate_recall(correct_words, pred_words):if len(correct_set) == 0:return True # Avoid division by zero if correct answer is somehow emptyrecall = len(common_words) / len(correct_set)- return recall >= 2/3+ return recall >= 2 / 3+def check_conditions(example, pred, trace=None, debug=False):category = example.category@@ -35,7 +40,10 @@ def check_conditions(example, pred, trace=None, debug=False):# Check for exact match after preprocessingif preprocessed_pred_answer != preprocessed_correct_answer:- if debug: print(f"Exact match failed. Expected: '{preprocessed_correct_answer}', Got: '{preprocessed_pred_answer}'")+ if debug:+ print(+ f"Exact match failed. Expected: '{preprocessed_correct_answer}', Got: '{preprocessed_pred_answer}'"+ )return False# When the answer is a place, the response should contain no punctuation@@ -43,7 +51,10 @@ def check_conditions(example, pred, trace=None, debug=False):if any(char in answer for char in ",.?!;:"):return Falseelse:- if debug: print(f"Place. When the answer is a place, the response should contain no punctuation {answer}")+ if debug:+ print(+ f"Place. When the answer is a place, the response should contain no punctuation {answer}"+ )return True# When the answer is a date, the response should end with "Peace!"@@ -51,7 +62,10 @@ def check_conditions(example, pred, trace=None, debug=False):if not answer.endswith("Peace!"):return Falseelse:- if debug: print(f"Date. When the answer is a date, the response should end with Peace! {answer}")+ if debug:+ print(+ f"Date. When the answer is a date, the response should end with Peace! {answer}"+ )return True# When the answer is a person, the response should be entirely in lowercase@@ -59,7 +73,10 @@ def check_conditions(example, pred, trace=None, debug=False):if answer != answer.lower():return Falseelse:- if debug: print(f"Answer. When the answer is a person, the response should be entirely in lowercase {answer}")+ if debug:+ print(+ f"Answer. When the answer is a person, the response should be entirely in lowercase {answer}"+ )return True# When the answer is none of the above categories, the response should be in all caps and not end with "Peace!"@@ -67,9 +84,13 @@ def check_conditions(example, pred, trace=None, debug=False):if answer != answer.upper() or answer.endswith("Peace!"):return Falseelse:- if debug: print(f"Other category. the response should be in all caps and not end with Peace! {answer}")+ if debug:+ print(+ f"Other category. the response should be in all caps and not end with Peace! {answer}"+ )return True+class MultiHop(dspy.Module):def __init__(self, passages_per_hop):super().__init__()@@ -87,6 +108,7 @@ class MultiHop(dspy.Module):answer=self.generate_answer(context=context, question=question).answer,)+class GenerateAnswerInstruction(dspy.Signature):"""When the answer is a person, respond entirely in lowercase. When the answer is a place, ensure your response contains no punctuation. When the answer is a date, end your response with “Peace!”. Never end your response with "Peace!" under other circumstances. When the answer is none of the above categories respond in all caps."""@@ -94,6 +116,7 @@ class GenerateAnswerInstruction(dspy.Signature):question = dspy.InputField(desc="Question we want an answer to")answer = dspy.OutputField(desc="Answer to the question")+class MultiHopHandwritten(dspy.Module):def __init__(self, passages_per_hop):super().__init__()@@ -111,17 +134,24 @@ class MultiHopHandwritten(dspy.Module):answer=self.generate_answer(context=context, question=question).answer,)+class HotPotQAConditionalTask(BaseTask):def __init__(self):# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqa# Get the directory where this script is locatedscript_dir = os.path.dirname(os.path.abspath(__file__))-+# Construct the absolute paths to the datasets- hotpotqa_train_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_train.csv")- hotpotqa_dev_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_dev.csv")- hotpotqa_test_path = os.path.join(script_dir, "../datasets/hotpotqa_conditional/hotpot_test.csv")+ hotpotqa_train_path = os.path.join(+ script_dir, "../datasets/hotpotqa_conditional/hotpot_train.csv"+ )+ hotpotqa_dev_path = os.path.join(+ script_dir, "../datasets/hotpotqa_conditional/hotpot_dev.csv"+ )+ hotpotqa_test_path = os.path.join(+ script_dir, "../datasets/hotpotqa_conditional/hotpot_test.csv"+ )# Read the datasetshotpotqa_train = pd.read_csv(hotpotqa_train_path)@@ -131,11 +161,25 @@ class HotPotQAConditionalTask(BaseTask):combined_train = pd.concat([hotpotqa_train, hotpotqa_dev], ignore_index=True)# Load and configure the datasets.- self.trainset = [dspy.Example(question=row['question'], answer=row['answer'], category=row['answer category']).with_inputs('question') for index, row in combined_train.iterrows()]- self.testset = [dspy.Example(question=row['question'], answer=row['answer'], category=row['answer category']).with_inputs('question') for index, row in hotpotqa_test.iterrows()]+ self.trainset = [+ dspy.Example(+ question=row["question"],+ answer=row["answer"],+ category=row["answer category"],+ ).with_inputs("question")+ for index, row in combined_train.iterrows()+ ]+ self.testset = [+ dspy.Example(+ question=row["question"],+ answer=row["answer"],+ category=row["answer category"],+ ).with_inputs("question")+ for index, row in hotpotqa_test.iterrows()+ ]# Set up metrics- NUM_THREADS=16+ NUM_THREADS = 16# TODO: set up metricsself.metric = check_conditions@@ -145,9 +189,7 @@ class HotPotQAConditionalTask(BaseTask):self.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)def get_program(self):- return MultiHop(- passages_per_hop=3- )+ return MultiHop(passages_per_hop=3)def get_metric(self):return self.metric@@ -157,6 +199,6 @@ class HotPotQAConditionalTask(BaseTask):def get_default_max_labeled_demos(self):return 0-+def get_max_tokens(self):return 600diff --git a/testing/tasks/hover.py b/testing/tasks/hover.pyindex c7e4c1161..1a7e4378c 100644--- a/testing/tasks/hover.py+++ b/testing/tasks/hover.py@@ -9,14 +9,27 @@ from dspy.evaluate import Evaluatefrom .base_task import BaseTask+def count_unique_docs(example):return len(set([fact["key"] for fact in example["supporting_facts"]]))+def discrete_retrieval_eval(example, pred, trace=None):- gold_titles = set(map(dspy.evaluate.normalize_text, [doc['key'] for doc in example['supporting_facts']]))- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.retrieved_docs]))+ gold_titles = set(+ map(+ dspy.evaluate.normalize_text,+ [doc["key"] for doc in example["supporting_facts"]],+ )+ )+ found_titles = set(+ map(+ dspy.evaluate.normalize_text,+ [c.split(" | ")[0] for c in pred.retrieved_docs],+ )+ )return gold_titles.issubset(found_titles)+class RetrieveMultiHop(dspy.Module):def __init__(self):super().__init__()@@ -27,47 +40,58 @@ class RetrieveMultiHop(dspy.Module):self.summarize1 = dspy.ChainOfThought("claim,passages->summary")self.summarize2 = dspy.ChainOfThought("claim,context,passages->summary")- def forward(self,claim):+ def forward(self, claim):# HOP 1hop1_docs = self.retrieve_k(claim).passages- summary_1 = self.summarize1(claim=claim, passages=hop1_docs).summary # Summarize top k docs-+ summary_1 = self.summarize1(+ claim=claim, passages=hop1_docs+ ).summary # Summarize top k docs+# HOP 2hop2_query = self.create_query_hop2(claim=claim, summary_1=summary_1).queryhop2_docs = self.retrieve_k(hop2_query).passages- summary_2 = self.summarize2(claim=claim, context=summary_1, passages=hop2_docs).summary+ summary_2 = self.summarize2(+ claim=claim, context=summary_1, passages=hop2_docs+ ).summary# HOP 3- hop3_query = self.create_query_hop3(claim=claim, summary_1=summary_1, summary_2=summary_2).query+ hop3_query = self.create_query_hop3(+ claim=claim, summary_1=summary_1, summary_2=summary_2+ ).queryhop3_docs = self.retrieve_k(hop3_query).passages- return dspy.Prediction(retrieved_docs = hop1_docs + hop2_docs + hop3_docs)+ return dspy.Prediction(retrieved_docs=hop1_docs + hop2_docs + hop3_docs)+class HoverRetrieveDiscrete(BaseTask):def __init__(self):dataset = load_dataset("hover")- hf_trainset = dataset['train']- hf_testset = dataset['validation']+ hf_trainset = dataset["train"]+ hf_testset = dataset["validation"]reformatted_hf_trainset = []reformatted_hf_testset = []for example in tqdm.tqdm(hf_trainset):- claim = example['claim']- supporting_facts = example['supporting_facts']- label = example['label']+ claim = example["claim"]+ supporting_facts = example["supporting_facts"]+ label = example["label"]- if count_unique_docs(example) == 3: # Limit to 3 hop examples- reformatted_hf_trainset.append(dict(claim=claim, supporting_facts=supporting_facts, label=label))+ if count_unique_docs(example) == 3: # Limit to 3 hop examples+ reformatted_hf_trainset.append(+ dict(claim=claim, supporting_facts=supporting_facts, label=label)+ )for example in tqdm.tqdm(hf_testset):- claim = example['claim']- supporting_facts = example['supporting_facts']- label = example['label']+ claim = example["claim"]+ supporting_facts = example["supporting_facts"]+ label = example["label"]if count_unique_docs(example) == 3:- reformatted_hf_testset.append(dict(claim=claim, supporting_facts=supporting_facts, label=label))+ reformatted_hf_testset.append(+ dict(claim=claim, supporting_facts=supporting_facts, label=label)+ )rng = random.Random(0)rng.shuffle(reformatted_hf_trainset)@@ -77,8 +101,8 @@ class HoverRetrieveDiscrete(BaseTask):trainset = reformatted_hf_trainsettestset = reformatted_hf_testset- self.trainset = [dspy.Example(**x).with_inputs('claim') for x in trainset]- self.testset = [dspy.Example(**x).with_inputs('claim') for x in testset]+ self.trainset = [dspy.Example(**x).with_inputs("claim") for x in trainset]+ self.testset = [dspy.Example(**x).with_inputs("claim") for x in testset]# Set up metricsNUM_THREADS = 16@@ -92,16 +116,15 @@ class HoverRetrieveDiscrete(BaseTask):def get_program(self):return RetrieveMultiHop()-+def get_metric(self):return self.metric-+def get_default_max_bootstrapped_demos(self):return 1def get_default_max_labeled_demos(self):return 2-+def get_max_tokens(self):return 600-diff --git a/testing/tasks/iris.py b/testing/tasks/iris.pyindex 64e0fcab6..ec0646f32 100644--- a/testing/tasks/iris.py+++ b/testing/tasks/iris.py@@ -4,6 +4,7 @@ from .base_task import BaseTaskimport randomimport dspy+class Sig(dspy.Signature):"Given the petal and sepal dimensions in cm, predict the iris species."@@ -11,30 +12,51 @@ class Sig(dspy.Signature):petal_width = dspy.InputField()sepal_length = dspy.InputField()sepal_width = dspy.InputField()- answer = dspy.OutputField(desc='setosa, versicolor, or virginica')+ answer = dspy.OutputField(desc="setosa, versicolor, or virginica")+class Classify(dspy.Module):def __init__(self):self.pred = dspy.ChainOfThought(Sig)-+def forward(self, petal_length, petal_width, sepal_length, sepal_width):- return self.pred(petal_length=petal_length, petal_width=petal_width, sepal_length=sepal_length, sepal_width=sepal_width)-+ return self.pred(+ petal_length=petal_length,+ petal_width=petal_width,+ sepal_length=sepal_length,+ sepal_width=sepal_width,+ )++class IrisClassifierTask(BaseTask):def __init__(self):# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqadataset = load_dataset("hitorilabs/iris")- fullset = [dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()}) for example in dataset['train']]- fullset = [dspy.Example(**{**x, 'answer': ['setosa', 'versicolor', 'virginica'][int(x['species'])]}) for x in fullset]- fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]+ fullset = [+ dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()})+ for example in dataset["train"]+ ]+ fullset = [+ dspy.Example(+ **{+ **x,+ "answer": ["setosa", "versicolor", "virginica"][int(x["species"])],+ }+ )+ for x in fullset+ ]+ fullset = [+ x.with_inputs("petal_length", "petal_width", "sepal_length", "sepal_width")+ for x in fullset+ ]random.Random(0).shuffle(fullset)# self.trainset, self.devset, self.testset = fullset[:25], fullset[20:75], fullset[75:]- self.trainset, self.testset = fullset[:75], fullset[75:]+ self.trainset, self.testset = fullset[:75], fullset[75:]# Set up metrics- NUM_THREADS=16+ NUM_THREADS = 16# TODO: set up metricsself.metric = dspy.evaluate.answer_exact_match@@ -60,12 +82,12 @@ class IrisClassifierTask(BaseTask):def get_trainset(self, TRAIN_NUM=None):return self.trainset-+def get_testset(self):return self.testsetdef get_devset(self, TRAIN_NUM=None, DEV_NUM=None):return self.trainset-+def get_evalset(self, EVAL_NUM=None):- return self.testset\ No newline at end of file+ return self.testsetdiff --git a/testing/tasks/iris_typo.py b/testing/tasks/iris_typo.pyindex ee0e6b3a6..9bff74ec6 100644--- a/testing/tasks/iris_typo.py+++ b/testing/tasks/iris_typo.py@@ -4,6 +4,7 @@ from .base_task import BaseTaskimport randomimport dspy+class Sig(dspy.Signature):"Given the petal and sepal dimensions in cm, predict the iris species."# "Given the petal and sepal dimensions in cm, predict the iris species. If petal_width < 1.0, setosa. Otherwise, if petal_width < 1.65, versicolor. Otherwise, virginica."@@ -12,29 +13,50 @@ class Sig(dspy.Signature):petal_width = dspy.InputField()sepal_length = dspy.InputField()sepal_width = dspy.InputField()- answer = dspy.OutputField(desc='setosa, versicolour, or virginica')+ answer = dspy.OutputField(desc="setosa, versicolour, or virginica")+class Classify(dspy.Module):def __init__(self):self.pred = dspy.ChainOfThought(Sig)-+def forward(self, petal_length, petal_width, sepal_length, sepal_width):- return self.pred(petal_length=petal_length, petal_width=petal_width, sepal_length=sepal_length, sepal_width=sepal_width)-+ return self.pred(+ petal_length=petal_length,+ petal_width=petal_width,+ sepal_length=sepal_length,+ sepal_width=sepal_width,+ )++class IrisTypoClassifierTask(BaseTask):def __init__(self):# Read in the conditional HotpotQA dataset from nfl_datasets as a csv from nfl_datasets/conditional_hotpotqadataset = load_dataset("hitorilabs/iris")- fullset = [dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()}) for example in dataset['train']]- fullset = [dspy.Example(**{**x, 'answer': ['setosa', 'versicolor', 'virginica'][int(x['species'])]}) for x in fullset]- fullset = [x.with_inputs('petal_length', 'petal_width', 'sepal_length', 'sepal_width') for x in fullset]+ fullset = [+ dspy.Example(**{k: str(round(v, 2)) for k, v in example.items()})+ for example in dataset["train"]+ ]+ fullset = [+ dspy.Example(+ **{+ **x,+ "answer": ["setosa", "versicolor", "virginica"][int(x["species"])],+ }+ )+ for x in fullset+ ]+ fullset = [+ x.with_inputs("petal_length", "petal_width", "sepal_length", "sepal_width")+ for x in fullset+ ]random.Random(0).shuffle(fullset)- self.trainset, self.testset = fullset[:75], fullset[75:]+ self.trainset, self.testset = fullset[:75], fullset[75:]# Set up metrics- NUM_THREADS=16+ NUM_THREADS = 16# TODO: set up metricsself.metric = dspy.evaluate.answer_exact_match@@ -60,12 +82,12 @@ class IrisTypoClassifierTask(BaseTask):def get_trainset(self, TRAIN_NUM=None):return self.trainset-+def get_testset(self):return self.testsetdef get_devset(self, TRAIN_NUM=None, DEV_NUM=None):return self.trainset-+def get_evalset(self, EVAL_NUM=None):- return self.testset\ No newline at end of file+ return self.testsetdiff --git a/testing/tasks/scone.py b/testing/tasks/scone.pyindex d0c5f5193..2c35dbf4b 100644--- a/testing/tasks/scone.py+++ b/testing/tasks/scone.py@@ -13,29 +13,32 @@ def load_scone(dirname):dfs = []for filename in glob.glob(dirname + "/*.csv"):df = pd.read_csv(filename, index_col=0)- df['category'] = os.path.basename(filename).replace(".csv", "")+ df["category"] = os.path.basename(filename).replace(".csv", "")dfs.append(df)data_df = pd.concat(dfs)def as_example(row):# The 'one_scoped' file is from an earlier dataset, MoNLI, and# so is formatted a bit differently:- suffix = '' if row['category'] == 'one_scoped' else '_edited'+ suffix = "" if row["category"] == "one_scoped" else "_edited"# Reformat the hypothesis to be an embedded clause in a question:- hkey = 'sentence2' + suffix- question = row[hkey][0].lower() + row[hkey][1: ].strip(".")+ hkey = "sentence2" + suffix+ question = row[hkey][0].lower() + row[hkey][1:].strip(".")question = f"Can we logically conclude for sure that {question}?"# Binary task formulation:- label = "Yes" if row['gold_label' + suffix] == 'entailment' else "No"- return dspy.Example({- "context": row['sentence1' + suffix],- "question": question,- "answer": label,- "category": row['category'],- }).with_inputs("context", "question")+ label = "Yes" if row["gold_label" + suffix] == "entailment" else "No"+ return dspy.Example(+ {+ "context": row["sentence1" + suffix],+ "question": question,+ "answer": label,+ "category": row["category"],+ }+ ).with_inputs("context", "question")return list(data_df.apply(as_example, axis=1).values)+class ScoNeSignature(dspy.Signature):("""context, question -> answer""")@@ -43,6 +46,7 @@ class ScoNeSignature(dspy.Signature):question = dspy.InputField()answer = dspy.OutputField(desc="Yes or No")+class ScoNeCoT(dspy.Module):def __init__(self):super().__init__()@@ -50,7 +54,8 @@ class ScoNeCoT(dspy.Module):def forward(self, context, question):return self.generate_answer(context=context, question=question)-++class ScoNeTask(BaseTask):def __init__(self):# !git clone https://github.com/selenashe/ScoNe.git@@ -64,7 +69,7 @@ class ScoNeTask(BaseTask):random.shuffle(all_train)# 1000 random train, 500 random dev:- self.trainset, self.testset = all_train[: 1000], all_train[1000: 1500]+ self.trainset, self.testset = all_train[:1000], all_train[1000:1500]metric_EM = dspy.evaluate.answer_exact_matchself.metric = metric_EM@@ -73,9 +78,9 @@ class ScoNeTask(BaseTask):def get_program(self):return ScoNeCoT()-+def get_metric(self):return self.metric-+def get_max_tokens(self):- return 200\ No newline at end of file+ return 200diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.pyindex 3e6c19a8c..7e366bb9b 100644--- a/testing/tasks/tweet.py+++ b/testing/tasks/tweet.py@@ -17,6 +17,7 @@ class TweetSignature(dspy.Signature):question = dspy.InputField()answer = dspy.OutputField(desc="Yes or No")+class TweetCoT(dspy.Module):def __init__(self):super().__init__()@@ -25,42 +26,51 @@ class TweetCoT(dspy.Module):def forward(self, context, question):return self.generate_answer(context=context, question=question)+class MultiHopTweet(dspy.Module):- def __init__(self,passages_per_hop):+ def __init__(self, passages_per_hop):super().__init__()- self.retrieve = dspy.Retrieve(k = passages_per_hop)+ self.retrieve = dspy.Retrieve(k=passages_per_hop)self.generate_query = dspy.ChainOfThought("context ,question->search_query")self.generate_answer = TweetCoT()-- def forward (self,question) :++ def forward(self, question):context = []for hop in range(2):- query = self.generate_query(context = context, question = question).search_query+ query = self.generate_query(context=context, question=question).search_querycontext += self.retrieve(query).passages- return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)+ return dspy.Prediction(+ context=context,+ answer=self.generate_answer(context=context, question=question).answer,+ )+# Define the signature for automatic assessments.class Assess(dspy.Signature):"""Assess the quality of a tweet along the specified dimension."""- context = dspy.InputField(desc='ignore if N/A')+ context = dspy.InputField(desc="ignore if N/A")assessed_text = dspy.InputField()assessment_question = dspy.InputField()assessment_answer = dspy.OutputField(desc="Yes or No")+@lru_cachedef load_models():load_dotenv() # This will load the .env file's variables- openai.api_key = os.environ.get('OPENAI_API_KEY')- openai.api_base = os.environ.get('OPENAI_API_BASE')- gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')+ openai.api_key = os.environ.get("OPENAI_API_KEY")+ openai.api_base = os.environ.get("OPENAI_API_BASE")+ gpt4T = dspy.OpenAI(model="gpt-3.5-turbo-1106", max_tokens=1000, model_type="chat")retrieve = dspy.Retrieve(k=5)return gpt4T, retrieve+METRIC = None++def metric(gold, pred, trace=None):-+gpt4T, retrieve = load_models()question, answer, tweet = gold.question, gold.answer, pred.answer@@ -68,45 +78,73 @@ def metric(gold, pred, trace=None):engaging = "Does the assessed text make for a self-contained, engaging tweet?"faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."- correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ correct = (+ f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ )correct = f"{correct} Does the assessed text above contain the gold answer?"-- with dspy.context(lm=gpt4T):- faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)- correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)- engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])+ with dspy.context(lm=gpt4T):+ faithful = dspy.Predict(Assess)(+ context=context, assessed_text=tweet, assessment_question=faithful+ )+ correct = dspy.Predict(Assess)(+ context="N/A", assessed_text=tweet, assessment_question=correct+ )+ engaging = dspy.Predict(Assess)(+ context="N/A", assessed_text=tweet, assessment_question=engaging+ )++ correct, engaging, faithful = (+ m.assessment_answer.split()[0].lower() == "yes"+ for m in [correct, engaging, faithful]+ )score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0if METRIC is not None:- if METRIC == 'correct': return correct- if METRIC == 'engaging': return engaging- if METRIC == 'faithful': return faithful-- if trace is not None: return score >= 3+ if METRIC == "correct":+ return correct+ if METRIC == "engaging":+ return engaging+ if METRIC == "faithful":+ return faithful++ if trace is not None:+ return score >= 3return score / 3.0+class TweetTask(BaseTask):def __init__(self):# Load the dataset.- dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)+ dataset = HotPotQA(+ train_seed=1,+ train_size=500,+ eval_seed=2023,+ dev_size=200,+ test_size=0,+ keep_details=True,+ )# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.- self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]- self.testset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]+ self.trainset = [+ x.without("id", "type").with_inputs("question") for x in dataset.train+ ]+ self.testset = [+ x.without("id", "type").with_inputs("question") for x in dataset.dev+ ]- self.trainset = [x.with_inputs('question') for x in dataset.train]- self.testset = [x.with_inputs('question') for x in dataset.dev]+ self.trainset = [x.with_inputs("question") for x in dataset.train]+ self.testset = [x.with_inputs("question") for x in dataset.dev]self.metric = metricself.set_splits(TRAIN_NUM=100, DEV_NUM=100, TEST_NUM=100)def get_program(self):- return MultiHopTweet(passages_per_hop=3) #TODO: make it so we can specify # of passages-+ return MultiHopTweet(+ passages_per_hop=3+ ) # TODO: make it so we can specify # of passages+def get_metric(self):return self.metric-diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.pyindex a1c6f0464..5eec5f057 100644--- a/testing/tasks/tweet_metric.py+++ b/testing/tasks/tweet_metric.py@@ -20,6 +20,7 @@ class TweetSignature(dspy.Signature):question = dspy.InputField()answer = dspy.OutputField(desc="Yes or No")+class TweetCoT(dspy.Module):def __init__(self):super().__init__()@@ -28,61 +29,89 @@ class TweetCoT(dspy.Module):def forward(self, context, question):return self.generate_answer(context=context, question=question)+class MultiHopTweet(dspy.Module):- def __init__(self,passages_per_hop):+ def __init__(self, passages_per_hop):super().__init__()- self.retrieve = dspy.Retrieve(k = passages_per_hop)+ self.retrieve = dspy.Retrieve(k=passages_per_hop)self.generate_query = dspy.ChainOfThought("context ,question->search_query")self.generate_answer = TweetCoT()-- def forward (self,question) :++ def forward(self, question):context = []for hop in range(2):- query = self.generate_query(context = context, question = question).search_query+ query = self.generate_query(context=context, question=question).search_querycontext += self.retrieve(query).passages- return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)+ return dspy.Prediction(+ context=context,+ answer=self.generate_answer(context=context, question=question).answer,+ )+# Define the signature for automatic assessments.class Assess(dspy.Signature):"""Assess the quality of a tweet along the specified dimension."""- context = dspy.InputField(desc='ignore if N/A')+ context = dspy.InputField(desc="ignore if N/A")assessed_text = dspy.InputField()assessment_question = dspy.InputField()assessment_answer = dspy.OutputField(desc="Yes or No")+@lru_cachedef load_models():load_dotenv() # This will load the .env file's variables- openai.api_key = os.environ.get('OPENAI_API_KEY')- openai.api_base = os.environ.get('OPENAI_API_BASE')- gpt3T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')- gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')+ openai.api_key = os.environ.get("OPENAI_API_KEY")+ openai.api_base = os.environ.get("OPENAI_API_BASE")+ gpt3T = dspy.OpenAI(model="gpt-3.5-turbo-1106", max_tokens=1000, model_type="chat")+ gpt4T = dspy.OpenAI(model="gpt-4-1106-preview", max_tokens=1000, model_type="chat")retrieve = dspy.Retrieve(k=5)return gpt3T, gpt4T, retrieve+METRIC = None++def metric(gold, pred, trace=None):gpt3T, gpt4T, retrieve = load_models()- question, answer, tweet, context = gold.question, gold.answer, gold.tweet, gold.context+ question, answer, tweet, context = (+ gold.question,+ gold.answer,+ gold.tweet,+ gold.context,+ )score_pred = pred.scoreengaging = "Does the assessed text make for a self-contained, engaging tweet?"faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."- correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ correct = (+ f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ )correct = f"{correct} Does the assessed text above contain the gold answer?"-- with dspy.context(lm=gpt3T): # TODO Update to GPT4- faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)- correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)- engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])+ with dspy.context(lm=gpt3T): # TODO Update to GPT4+ faithful = dspy.Predict(Assess)(+ context=context, assessed_text=tweet, assessment_question=faithful+ )+ correct = dspy.Predict(Assess)(+ context="N/A", assessed_text=tweet, assessment_question=correct+ )+ engaging = dspy.Predict(Assess)(+ context="N/A", assessed_text=tweet, assessment_question=engaging+ )++ correct, engaging, faithful = (+ m.assessment_answer.split()[0].lower() == "yes"+ for m in [correct, engaging, faithful]+ )score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0- return 1 - abs(score - score_pred) # We want a score we can maximize, so take the negative L1 norm and add 1+ return 1 - abs(+ score - score_pred+ ) # We want a score we can maximize, so take the negative L1 norm and add 1+class TweetMetric(dspy.Module):def __init__(self):@@ -90,31 +119,54 @@ class TweetMetric(dspy.Module):self.engaging = dspy.Predict(Assess)self.faithful = dspy.Predict(Assess)self.correct = dspy.Predict(Assess)-- def forward (self, tweet, context, question, answer) :++ def forward(self, tweet, context, question, answer):engaging = "Does the assessed text make for a self-contained, engaging tweet?"faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."correct = f"{correct} Does the assessed text above contain the gold answer?"-- faithful = self.faithful(context=context, assessed_text=tweet, assessment_question=faithful)- correct = self.correct(context='N/A', assessed_text=tweet, assessment_question=correct)- engaging = self.engaging(context='N/A', assessed_text=tweet, assessment_question=engaging)- correct, engaging, faithful = (m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful])- score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0+ faithful = self.faithful(+ context=context, assessed_text=tweet, assessment_question=faithful+ )+ correct = self.correct(+ context="N/A", assessed_text=tweet, assessment_question=correct+ )+ engaging = self.engaging(+ context="N/A", assessed_text=tweet, assessment_question=engaging+ )++ correct, engaging, faithful = (+ m.assessment_answer.split()[0].lower() == "yes"+ for m in [correct, engaging, faithful]+ )+ score = (+ (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0+ )++ return dspy.Prediction(score=score / 3.0)- return dspy.Prediction(score= score/3.0)class TweetMetricTask(BaseTask):def __init__(self):# Load the dataset.- dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)+ dataset = HotPotQA(+ train_seed=1,+ train_size=500,+ eval_seed=2023,+ dev_size=200,+ test_size=0,+ keep_details=True,+ )# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.- trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]- devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]+ trainset_temp = [+ x.without("id", "type").with_inputs("question") for x in dataset.train+ ]+ devset_temp = [+ x.without("id", "type").with_inputs("question") for x in dataset.dev+ ]self.trainset = []self.testset = []@@ -126,18 +178,36 @@ class TweetMetricTask(BaseTask):question = ex.questionanswer = ex.answertweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer- example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}-- self.trainset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='train').with_inputs('context', 'question', 'answer', 'tweet'))+ example = {+ "context": context,+ "question": question,+ "answer": answer,+ "tweet": tweet,+ }++ self.trainset.append(+ Example(+ **example, dspy_uuid=str(uuid.uuid4()), dspy_split="train"+ ).with_inputs("context", "question", "answer", "tweet")+ )for ex in tqdm(devset_temp, desc="Preprocessing Devset"):context = retrieve(ex.question).passagesquestion = ex.questionanswer = ex.answertweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer- example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}-- self.testset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))+ example = {+ "context": context,+ "question": question,+ "answer": answer,+ "tweet": tweet,+ }++ self.testset.append(+ Example(+ **example, dspy_uuid=str(uuid.uuid4()), dspy_split="dev"+ ).with_inputs("context", "question", "answer", "tweet")+ )self.metric = metric@@ -145,7 +215,6 @@ class TweetMetricTask(BaseTask):def get_program(self):return TweetMetric()-+def get_metric(self):return self.metric-