DSPy: The framework for programming—not prompting—language models
Python
Added optimizer testing framework
Michael Ryan285eebcd9494
diff --git a/testing/README.md b/testing/README.mdnew file mode 100644index 000000000..60ed5025e--- /dev/null+++ b/testing/README.md@@ -0,0 +1,91 @@+# Optimizer Tester++Optimizer Tester is intended to allow simple and repeatable testing of DSPy Optimizers. This is a development tool for the creation of more optimizers, and the validation that they work across tasks. This is still under development and subject to change.++## Usage++To use the Optimizer Tester in code instantiate an OptimizerTester object:++```python+from testing.optimizer_tester import OptimizerTester++tester = OptimizerTester()+```++The default verison (no parameters) expects a llama model hosted on ports [7140, 7141, 7142, 7143] and OpenAI keys stored in a .env file (OPENAI_API_KEY and OPENAI_API_BASE).++If you prefer to specify your own model parameters then you can pass models into the OptimizerTester++```python+task_model = dspy.HFClientTGI(...)+prompt_model = dspy.OpenAI(...)++tester = OptimizerTester()+```++If you just want to get baseline results for a particular task you're ready to go!++```python+tester.test_baseline(datasets=["HotPotQA", "GSM8K", "Scone"])+```++If you want to test out a custom optimizer you'll have to write a quick function to call it properly:++```python+def your_optimizer_caller(default_program, trainset, devset, test_name, dataset_name, kwargs):+ # initialize your teleprompter (optimizer) here!+ teleprompter = my_custom_optimizer(metric=kwargs["metric"], ...)++ # call your optimizer on the default_program here!+ compiled_program = teleprompter.compile(default_program.deepcopy(), trainset=trainset, ...)++ # if you wish to tweak any of the outputs to the csv file you can do that here+ output = {+ "test_name": "my_optimizer-" + dataset_name + "-" + test_name+ }++ # return the compiled program and modified output (or empty dict if no changes made)+ return compiled_program, output+```++You can use these kwargs:+- breadth=self.BREADTH+- depth=self.DEPTH+- temperature=self.TEMPERATURE+- prompt_model=self.prompt_model+- view_data=False+- log_dir=log_dir+- metric=task.get_metric()+- task_model=self.task_model++These are somewhat specific to the testing for the signature optimizer teleprompter which this tool was originally built for, so you probably will not use most of these.++Here is an example of an implemented version of the function:++```python+def signature_optimizer_default(default_program, trainset, devset, test_name, dataset_name, kwargs):+ eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)++ teleprompter = SignatureOptimizer(prompt_model=kwargs["prompt_model"], task_model=kwargs["task_model"], metric=kwargs["metric"], breadth=kwargs["breadth"], depth=kwargs["depth"], init_temperature=kwargs["temperature"], verbose=False, log_dir=kwargs["log_dir"])+ compiled_program = teleprompter.compile(default_program.deepcopy(), devset=trainset, evalset=devset, eval_kwargs=eval_kwargs)++ output = {+ "meta_prompt_style": "best",+ "view_data": True,+ "test_name": dataset_name + "_" + test_name+ }++ return compiled_program, output+```++Then you can use the 'test_optimizer_default' test to run the tests on your optimizer:++```python+tester.test_optimizer_default(signature_optimizer_default, datasets=["BioDex", "Tweet", "Tweet Metric"])+```++We are working on more optimizer tests with interesting compositions of optimizers, and other hyperparameter adjustments, which will be added as they are developed.++## Important Note++The BioDex Task requires an external download. Please see the `/tasks/biodex.py` file for a link to download details and a note about where to insert the download path in the code.\ No newline at end of filediff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.pynew file mode 100644index 000000000..ffcf3b262--- /dev/null+++ b/testing/optimizer_tester.py@@ -0,0 +1,204 @@+from .tasks.scone import ScoNeTask+from .tasks.hotpotqa import HotPotQATask+from .tasks.gsm8k import GSM8KTask+from .tasks.biodex import BioDexTask+from .tasks.tweet import TweetTask+from .tasks.tweet_metric import TweetMetricTask+from dspy.evaluate import Evaluate+from dotenv import load_dotenv+import openai+import os+import dspy+import csv+import datetime+from timeit import default_timer as timer++datasets = ["ScoNe", "HotPotQA", "GSM8K", "BioDex", "Tweet"]++class OptimizerTester:+ def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,+ num_threads = 10, default_breadth = 10, default_depth = 3, default_temperature = 1.4,+ prompt_model_name = "gpt-3.5-turbo-1106", task_model_name = "meta-llama/Llama-2-13b-chat-hf",+ prompt_model = None, task_model = None,+ colbert_v2_endpoint = "http://20.102.90.50:2017/wiki17_abstracts"):+ self.datasets = datasets+ self.TRAIN_NUM = default_train_num+ self.DEV_NUM = default_dev_num+ self.EVAL_NUM = default_test_num+ self.NUM_THREADS = num_threads+ self.BREADTH = default_breadth+ self.DEPTH = default_depth+ self.TEMPERATURE = default_temperature+ self.PROMPT_MODEL_NAME = prompt_model_name+ self.TASK_MODEL_NAME = task_model_name+ self.COLBERT_V2_ENDPOINT = colbert_v2_endpoint++ load_dotenv() # This will load the .env file's variables++ openai.api_key = os.environ.get('OPENAI_API_KEY')+ openai.api_base = os.environ.get('OPENAI_API_BASE')++ # Prompt gen model+ if not prompt_model:+ self.prompt_model = dspy.OpenAI(model=self.PROMPT_MODEL_NAME, max_tokens=150)+ else:+ self.prompt_model = prompt_model++ # Task model+ if not task_model:+ self.task_model = dspy.HFClientTGI(model=self.TASK_MODEL_NAME, port=[7140, 7141, 7142, 7143], max_tokens=150)+ else:+ self.task_model = task_model+ self.colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)++ dspy.settings.configure(rm=self.colbertv2, lm=self.task_model)++ def write_to_csv(self, folder_name, file_name, data):+ # Ensure the output directory exists+ os.makedirs(folder_name, exist_ok=True)+ file_path = os.path.join(folder_name, file_name)++ # Check if file exists to determine if headers should be written+ file_exists = os.path.isfile(file_path)++ with open(file_path, mode='a', newline='', encoding='utf-8') as file:+ writer = csv.writer(file)++ headers = ['test_name', 'train_score', 'dev_score', 'eval_score', 'run_time (sec)', 'train_size', 'dev_size', 'eval_size', 'task_name', 'signature_optimized', 'prompt_model_name', 'task_model_name', 'breadth', 'depth', 'meta_prompt_style', 'fewshot_before', 'fewshot_after', 'temperature', 'fewshot_candidates_num', 'max_bootstrapped_demos', 'bootstrapping', 'view_data', 'optimizer_log_dir', 'additional_notes', 'misc']++ # Write headers if the file is being created+ if not file_exists:+ writer.writerow(headers)++ # Format the data+ formatted_data = ['NA'] * len(headers)+ formatted_data[-1] = ""+ for key in data.keys():+ if key in headers:+ formatted_data[headers.index(key)] = data[key]+ else:+ formatted_data[-1] += f"{key}: {data[key]}\n"+ # Write the data+ writer.writerow(formatted_data)+++ def load_dataset(self,dataset):+ ds = None+ dataset = dataset.lower()+ if dataset == "scone":+ ds = ScoNeTask()+ elif dataset == "hotpotqa":+ ds = HotPotQATask()+ elif dataset == "gsm8k":+ ds = GSM8KTask()+ elif dataset == "biodex":+ ds = BioDexTask()+ elif dataset == "tweet":+ ds = TweetTask()+ elif dataset == "tweet_metric":+ ds = TweetMetricTask()+ else:+ raise ValueError("Invalid dataset name.")+ ds.set_splits(TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, EVAL_NUM=self.EVAL_NUM)+ return ds++ # Computes baseline results for a given dataset+ def test_baseline(self, datasets=datasets, test_name="baseline"):+ for dataset in datasets:+ task = self.load_dataset(dataset)+ evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ evaluate_eval = Evaluate(devset=task.get_evalset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ default_program = task.get_program()++ # Evaluate the default program+ default_results_train = evaluate_train(default_program)+ default_results_dev = evaluate_dev(default_program)+ default_results_eval = evaluate_eval(default_program)++ # Write the results to a csv+ self.write_to_csv('outputs', 'results.csv', {+ 'test_name': dataset + "_" + test_name,+ 'train_score': default_results_train,+ 'dev_score': default_results_dev,+ 'eval_score': default_results_eval,+ 'run_time (sec)': 0,+ 'train_size': len(task.get_trainset()),+ 'dev_size': len(task.get_devset()),+ 'eval_size': len(task.get_evalset()),+ 'task_name': dataset,+ 'signature_optimized': False,+ 'prompt_model_name': self.PROMPT_MODEL_NAME,+ 'task_model_name': self.TASK_MODEL_NAME,+ 'breadth': 'NA',+ 'depth': 'NA',+ 'meta_prompt_style': 'default',+ 'fewshot_before': False,+ 'fewshot_after': False,+ 'temperature': self.TEMPERATURE,+ 'fewshot_candidates_num': 0,+ 'max_bootstrapped_demos': 0,+ 'bootstrapping': False,+ 'view_data': False,+ 'optimizer_log_dir': 'NA',+ 'additional_notes': '',+ 'misc': ''+ })++ def test_optimizer_default(self, optimizer_function, datasets=datasets, test_name="default"):++ for dataset in datasets:+ task = self.load_dataset(dataset)+ evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ evaluate_eval = Evaluate(devset=task.get_evalset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)+ default_program = task.get_program()++ # Set up the optimizer kwargs+ date_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")+ log_dir = "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"+ os.makedirs(log_dir, exist_ok=True)+ kwargs = dict(breadth=self.BREADTH, depth=self.DEPTH, temperature=self.TEMPERATURE, prompt_model=self.prompt_model, view_data=False, log_dir=log_dir, metric=task.get_metric(), task_model=self.task_model)++ # Optimize the default program+ start = timer()+ optimized_program, output_dict = optimizer_function(default_program, task.get_trainset(), task.get_devset(), test_name, dataset, kwargs)+ end = timer()++ # Evaluate the optimized program+ optimized_results_train = evaluate_train(optimized_program)+ optimized_results_dev = evaluate_dev(optimized_program)+ optimized_results_eval = evaluate_eval(optimized_program)++ output = {+ 'test_name': dataset + "_" + test_name,+ 'train_score': optimized_results_train,+ 'dev_score': optimized_results_dev,+ 'eval_score': optimized_results_eval,+ 'run_time (sec)': end - start,+ 'train_size': len(task.get_trainset()),+ 'dev_size': len(task.get_devset()),+ 'eval_size': len(task.get_evalset()),+ 'task_name': dataset,+ 'signature_optimized': True,+ 'prompt_model_name': self.PROMPT_MODEL_NAME,+ 'task_model_name': self.TASK_MODEL_NAME,+ 'breadth': self.BREADTH,+ 'depth': self.DEPTH,+ 'meta_prompt_style': 'default',+ 'fewshot_before': False,+ 'fewshot_after': False,+ 'temperature': self.TEMPERATURE,+ 'fewshot_candidates_num': 0,+ 'max_bootstrapped_demos': 0,+ 'bootstrapping': False,+ 'view_data': False,+ 'optimizer_log_dir': log_dir,+ 'additional_notes': '',+ 'misc': ''+ }++ output.update(output_dict)++ # Write the results to a csv+ self.write_to_csv('outputs', 'results.csv', output)\ No newline at end of filediff --git a/testing/tasks/__init__.py b/testing/tasks/__init__.pynew file mode 100644index 000000000..f5034e36f--- /dev/null+++ b/testing/tasks/__init__.py@@ -0,0 +1,7 @@+from .base_task import BaseTask+from .gsm8k import GSM8KTask+from .hotpotqa import HotPotQATask+from .scone import ScoNeTask+from .biodex import BioDexTask+from .tweet import TweetTask+from .tweet_metric import TweetMetricTask\ No newline at end of filediff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.pynew file mode 100644index 000000000..c7f8f42b6--- /dev/null+++ b/testing/tasks/base_task.py@@ -0,0 +1,48 @@+from abc import ABC, abstractmethod++class BaseTask(ABC):+ def __init__(self):+ pass++ @abstractmethod+ def get_program(self):+ pass++ @abstractmethod+ def get_metric(self):+ pass++ def get_trainset(self, TRAIN_NUM=None):+ if TRAIN_NUM:+ self.TRAIN_NUM = TRAIN_NUM+ return self.trainset[:TRAIN_NUM]+ else:+ return self.trainset[:self.TRAIN_NUM]++ def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):+ if TRAIN_NUM:+ self.TRAIN_NUM = TRAIN_NUM+ if DEV_NUM:+ self.DEV_NUM = DEV_NUM++ index = -1+ if DEV_NUM and TRAIN_NUM:+ index = max(len(self.trainset) - DEV_NUM, TRAIN_NUM)+ else:+ index = max(len(self.trainset) - self.DEV_NUM, self.TRAIN_NUM)+ return self.trainset[index:]++ def get_evalset(self, EVAL_NUM=None):+ if EVAL_NUM:+ self.EVAL_NUM = EVAL_NUM+ return self.devset[:EVAL_NUM]+ else:+ return self.devset[:self.EVAL_NUM]++ def set_splits(self, TRAIN_NUM=None, DEV_NUM=None, EVAL_NUM=None):+ if TRAIN_NUM:+ self.TRAIN_NUM = TRAIN_NUM+ if DEV_NUM:+ self.DEV_NUM = DEV_NUM+ if EVAL_NUM:+ self.EVAL_NUM = EVAL_NUM\ No newline at end of filediff --git a/testing/tasks/biodex.py b/testing/tasks/biodex.pynew file mode 100644index 000000000..43b2d2e4a--- /dev/null+++ b/testing/tasks/biodex.py@@ -0,0 +1,633 @@+from __future__ import annotations+from .base_task import BaseTask+import sys+import dspy+from dspy.evaluate import Evaluate+from dsp.utils import deduplicate+import tqdm+import datasets+import sys+import math+from functools import lru_cache+import os+import re+import pickle+from pydantic import BaseModel+from rapidfuzz import process+from typing import Union, Optional, Dict, DefaultDict, List+from enum import Enum+from collections import defaultdict++# Must point to a MedDra download with mdhier.asc+data_dir = '/future/u/okhattab/data/2023/MedDraV2/meddra_23_0_english/MedAscii' # NOTE: EDIT THIS LINE++global meddra_root+meddra_root = None++def _remove_non_llt(candidates):+ return [c for c in candidates if c.level == Level.LLT]+++def get_path(node):+ path = f"under {node.parent.term} < {node.parent.parent.term} < {node.parent.parent.parent.term}"+ return path+++def flatten(ll):+ return [x for l in ll for x in l]+++def _matches_to_terms_and_paths(matches):+ s = []+ for n in matches:+ new_s = f"{n.node.term} ({n.score} score for query '{n.query}'; {n.node.count} counts; {get_path(n.node)})"+ s.append(new_s)+ return s+++class Level(Enum):+ ROOT = 0+ SOC = 1+ HLTG = 2+ HLT = 3+ LLT = 4++ def __add__(self, other: int):+ if self.ROOT.value <= self.value + other <= self.LLT.value:+ return Level(self.value + other)+ else:+ return None++ def __lt__(self, other) -> bool:+ return self.value < other.value++ def __le__(self, other) -> bool:+ return self.value <= other.value++ def __gt__(self, other) -> bool:+ return self.value > other.value++ def __ge__(self, other) -> bool:+ return self.value >= other.value++ def __sub__(self, other) -> int:+ return self.value - other.value+++class Node(BaseModel):+ id: str+ term: str+ level: Level+ count: int = 0++ parent: 'Node' = None+ children: Dict[str, 'Node'] = {}++ # maps for efficient nested children lookups+ # typically only used for the root node+ id_to_nodes: Optional[DefaultDict[str, List['Node']]] = None+ term_to_nodes: Optional[DefaultDict[str, List['Node']]] = None++ # text normalizer to build lookup tables+ # typically only used for the root node+ normalizer: function = lambda x: x++ def get_parent_at_level(self, target_level: Level) -> Union['Node', None]:+ # check if current node is deeper than target_level+ level_diff = target_level - self.level+ if level_diff > 0:+ return None+ else:+ node = self+ for _ in range(-level_diff):+ node = node.parent+ return node++ def get_children_at_level(self, target_level: Level) -> List['Node']:+ raise NotImplementedError++ def lookup_id(self, id: str) -> Union[List['Node'], None]:+ # create lookup tables if they do not exists+ if self.id_to_nodes == None:+ self.set_lookup_tables()+ # lookup++ if id in self.id_to_nodes:+ return self.id_to_nodes[id]+ else:+ return None++ def lookup_term(self, term: str) -> Union[List['Node'], None]:+ # create lookup tables if they do not exists+ if self.term_to_nodes == None:+ raise RuntimeError("First set the lookup tables using 'set_lookup_tables'.")+ # lookup+ term = self.normalizer(term)+ if term in self.term_to_nodes:+ return self.term_to_nodes[term]+ else:+ return None++ def match(+ self,+ query: str,+ fuzzy_limit: int = 20,+ fuzzy_score_cutoff: int = 80,+ return_nodes: bool = False,+ ):+ """Return a tuple containing exact match and best fuzzy matches of LLT terms in ontology.++ Args:+ query (str): String being matched into ontology+ fuzzy_limit (int, optional): Maximal limit of fuzzy matches to return. Defaults to 20.+ fuzzy_score_cutoff (int, optional): Minimal fuzzy matching score, between 0 and 100. Defaults to 80.+ return_nodes (bool, optional): If true, return Node objects instead of paths++ Returns:+ tuple: Tuple(exact match or None, list of fuzzy matches)+ """+ exact_candidate = []+ fuzzy_candidates = []++ # exact match+ exact_matches = self.lookup_term(query)+ if exact_matches is not None:+ exact_candidate = _remove_non_llt(exact_matches)+ exact_candidate = [+ Match(query=query, score=100, node=node) for node in exact_candidate+ ]++ # fuzzy match+ fuzzy_terms = process.extract(+ query,+ self.term_to_nodes.keys(),+ limit=fuzzy_limit,+ score_cutoff=fuzzy_score_cutoff,+ )++ fuzzy_matches = []+ for term, sim, _ in fuzzy_terms:+ applicable_nodes = _remove_non_llt(self.term_to_nodes[term])+ for n in applicable_nodes:+ fuzzy_matches.append(Match(node=n, score=sim, query=query))++ if return_nodes:+ return exact_candidate, fuzzy_matches+ else:+ exact_term = (+ _matches_to_terms_and_paths(exact_candidate) if exact_candidate else []+ )+ fuzzy_terms = _matches_to_terms_and_paths(fuzzy_matches)+ return exact_term, fuzzy_terms++ def set_lookup_tables(self, normalizer: function = lambda x: x) -> None:+ # set normalizer+ self.normalizer = normalizer++ # get a flat list of all nodes+ self.term_to_nodes = defaultdict(list)+ self.id_to_nodes = defaultdict(list)++ # breadth-first traversal+ candidates = [self]+ while len(candidates):+ candidate = candidates.pop(0)+ # track nodes+ self.term_to_nodes[self.normalizer(candidate.term)].append(candidate)+ self.id_to_nodes[candidate.id].append(candidate)+ # add candidate's children to candidate list+ if candidate.children:+ candidates.extend(candidate.children.values())+ return None++ def is_equivalent_node(self, other: 'Node') -> bool:+ # return true if two nodes are the same, siblings or in a parent-child relation+ return (+ (self == other)+ or (self.parent == other)+ or (self == other.parent)+ or (self.parent == other.parent)+ )++ def terms_equivalent(self, term1: str, term2: str) -> Union[bool, None]:+ # lookup both terms+ term1_nodes = self.lookup_term(term1)+ term2_nodes = self.lookup_term(term2)+ # if at least one term not found+ if not term1_nodes or not term2_nodes:+ return None+ # compare all nodes+ for node1 in term1_nodes:+ for node2 in term2_nodes:+ if node1.is_equivalent_node(node2):+ return True+ return False++ def __key(self):+ return (self.id, self.term, self.parent, len(self.children))++ def __hash__(self):+ return hash(self.__key())++ def __eq__(self, other: 'Node') -> bool:+ # undeep equals operator, does not check children+ return self.__key() == other.__key()++ def __str__(self) -> str:+ return f"Node(id={self.id}, term={self.term}, level={self.level}, parent={self.parent.term if self.parent else None}, #children={len(self.children)}, count={self.count})"++ def __repr__(self) -> str:+ return self.__str__()++ def get_path(self):+ path = f"under {self.parent.term} < {self.parent.parent.term} < {self.parent.parent.parent.term}"+ return path++ @property+ def all_nodes(self):+ return flatten(list(self.id_to_nodes.values()))++ @property+ def all_llt_nodes(self):+ return _remove_non_llt(self.all_nodes)+++class Match(BaseModel):+ node: Node+ score: int+ query: str++def _create_child_if_not_exists(+ parent: Node, child_id: str, child_term: str, term_to_count: dict+):+ parent_level = parent.level+ # get child level if still appropriate+ child_level = parent_level + 1+ if child_level is None:+ return None+ # get child if exists+ if child_id in parent.children:+ child = parent.children[child_id]+ # create child if not exists+ else:+ child_term_dict = child_term.lower().replace("'", "^")+ if child_term_dict in term_to_count:+ count = term_to_count[child_term_dict]+ else:+ count = 0+ child = Node(+ id=child_id, term=child_term, level=child_level, parent=parent, count=count+ )+ parent.children.update({child_id: child})+ return child++def parse_mdhier(data_dir):+ file = os.path.join(data_dir, "mdhier.asc")+ if not os.path.isfile(file):+ raise FileNotFoundError(f"File {file} not found.")+ count_file = os.path.join(data_dir, "priors.pickle")+ if not os.path.isfile(count_file):+ raise FileNotFoundError(f"File {count_file} not found.")++ # get regex+ id_regex = r"\d{8}"+ term_regex = r"[^$]*"+ regex = rf"({id_regex})\$({id_regex})\$({id_regex})\$({id_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$\$({id_regex})\$[YN]\$"+ regex = re.compile(regex)++ # get data+ lines = []+ with open(file, "r") as fp:+ lines = fp.readlines()+ text = "\n".join(lines)++ matches = regex.findall(text)++ print(f"Read {len(lines)} lines.")+ if len(lines) != len(matches):+ raise RuntimeError(+ f"Error parsing mdhier.asc. Expected number of matches to match number of lines."+ )++ # load counts+ with open(count_file, "rb") as fp:+ counts = pickle.load(fp)++ # create ontology tree+ root = Node(id="root", term="root", level=Level.ROOT)+ for match in matches:+ (+ llt_id,+ hlt_id,+ hltg_id,+ soc_id,+ llt_term,+ hlt_term,+ hltg_term,+ soc_term_1,+ soc_term_2,+ _,+ ) = match+ # recursivelty add the nodes specified by this line+ soc_node = _create_child_if_not_exists(root, soc_id, soc_term_1, counts)+ hltg_node = _create_child_if_not_exists(soc_node, hltg_id, hltg_term, counts)+ hlt_node = _create_child_if_not_exists(hltg_node, hlt_id, hlt_term, counts)+ llt_node = _create_child_if_not_exists(hlt_node, llt_id, llt_term, counts)++ # print(llt_node.parent == hlt_node)+ # print(hlt_node.parent == hltg_node)+ # print(hltg_node.parent == soc_node)+ # print(soc_node.parent == root)++ # print(Level.SOC < Level.LLT)+ # print(Level.SOC > Level.LLT)+ # print(Level.SOC - Level.LLT)++ # print(soc_node.get_parent_at_level(Level.ROOT))+ # print(soc_node.get_parent_at_level(Level.SOC))+ # print(soc_node.get_parent_at_level(Level.LLT))+ # print(llt_node.get_parent_at_level(Level.SOC))++ # print(root.lookup_term("COVID-19 pneumonia"))+ # print(root.lookup_term("COVID-19 pneumonia")[0].parent)+ return root++@lru_cache(maxsize=100000)+def base_ground(reaction, K):+ global meddra_root+ kwargs = dict(fuzzy_limit=K, fuzzy_score_cutoff=75, return_nodes=True)+ exact_matches, fuzzy_matches = meddra_root.match(reaction, **kwargs)++ return exact_matches, fuzzy_matches+++@lru_cache(maxsize=100000)+def ground_v1(reaction, K=3):+ """+ Prefers exact matches over fuzzy matches, when available.+ """++ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = exact_matches[:1] or fuzzy_matches+ matches = [(match.score / 100.0, match.node.term) for match in matches]+ matches = sorted(list(set(matches)), reverse=True)++ return tuple(matches)++@lru_cache(maxsize=100000)+def ground_v2(reaction, K=1):+ """+ When K=1 (default), returns exact matches (if available) or the best fuzzy match.+ """++ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = [(match.score / 100.0, match.node.term) for match in (exact_matches[:1] + fuzzy_matches)]+ matches = sorted(list(set(matches)), reverse=True)[:K]++ return tuple(matches)++# Returns the best three matches (including one exact match, if available).+ground_v3 = lambda reaction: ground_v2(reaction, K=3)++@lru_cache(maxsize=100000)+def ground_v4(reaction, K=3):+ """+ Returns the best three matches (including one exact match, if available) and applies a prior.+ """+ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = [((match.score / 100.0) * math.sqrt(match.node.count + 0.1), match.node.term)+ for match in (exact_matches[:1] + fuzzy_matches)]+ matches = sorted(list(set(matches)), reverse=True)[:K]++ return tuple(matches)++@lru_cache(maxsize=100000)+def ground_v4b(reaction, K=3):+ """+ Returns the best three matches (including one exact match, if available) and applies a prior.+ """++ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = [((match.score / 100.0) * math.log(match.node.count + 0.1), match.node.term)+ for match in (exact_matches[:1] + fuzzy_matches)]+ matches = sorted(list(set(matches)), reverse=True)[:K]++ return tuple(matches)++@lru_cache(maxsize=100000)+def ground_v4c(reaction, K=3):+ """+ Returns the best three matches (including one exact match, if available) and applies a prior.+ """++ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = [((match.score / 100.0) * (match.node.count + 0.1), match.node.term)+ for match in (exact_matches[:1] + fuzzy_matches)]+ matches = sorted(list(set(matches)), reverse=True)[:K]++ return tuple(matches)+++@lru_cache(maxsize=100000)+def ground_v5(reaction, K=3):+ exact_matches, fuzzy_matches = base_ground(reaction, K)++ matches = [((match.score / 100.0)**2 * math.log(match.node.count + 0.1), match.node.term)+ for match in (exact_matches[:1] + fuzzy_matches)]+ matches = sorted(list(set(matches)), reverse=True)[:K]++ return tuple(matches)++def preprocess_example(example):+ title = example['title']+ abstract = example['abstract']+ context = example['fulltext_processed'].split('\n\nTEXT:\n', 1)[-1]+ reactions = [r.strip().lower() for r in example['reactions'].split(',')]++ example = dict(title=title, abstract=abstract, context=context, reactions=reactions)+ example['labels'] = dspy.Example(reactions=reactions)++ return example++class Chunker:+ def __init__(self, context_window=3000, max_windows=5):+ self.context_window = context_window+ self.max_windows = max_windows+ self.window_overlap = 0.02++ def __call__(self, paper):+ snippet_idx = 0++ while snippet_idx < self.max_windows and paper:+ endpos = int(self.context_window * (1.0 + self.window_overlap))+ snippet, paper = paper[:endpos], paper[endpos:]++ next_newline_pos = snippet.rfind('\n')+ if paper and next_newline_pos != -1 and next_newline_pos >= self.context_window // 2:+ paper = snippet[next_newline_pos+1:] + paper+ snippet = snippet[:next_newline_pos]++ yield snippet_idx, snippet.strip()+ snippet_idx += 1++def get_set_precision_and_recalls(l1, l2):+ # lower case and strip+ l1 = [x.strip().lower() for x in l1]+ l2 = [x.strip().lower() for x in l2]++ s1, s2 = set(l1), set(l2)+ intersect = s1.intersection(s2)++ p, r = len(intersect) / max(1, len(s1)), len(intersect) / max(1, len(s2))+ f1 = 2 * (p * r) / max((p + r), 1)++ return dspy.Prediction(precision=p, recall=r, f1=f1)++def finegrained_metrics(gold, pred):+ reaction_metrics = get_set_precision_and_recalls(pred.reactions, gold.reactions)+ return reaction_metrics+++class CompilationMetric:+ __name__ = 'compilation_metric'++ def __init__(self, field='reactions', metric='recall', perfect_precision=False):+ self.field = field+ self.metric = metric+ self.perfect_precision = perfect_precision++ def __call__(self, gold, pred, trace=None):+ field = self.field+ metric = finegrained_metrics(gold, pred) #[field]++ if trace is None: # eval+ return metric[self.metric]++ if metric.precision >= 0.5 and metric.recall >= 0.5:+ for trace_idx in range(len(trace)):+ *_, outputs = trace[trace_idx]++ values = outputs[field].split(', ')+ values = [r.lower().strip('.').strip() for r in values]+ values = deduplicate([r for r in values if r and r != 'n/a'])++ if self.perfect_precision:+ values = [r for r in values if r in gold[self.field]]++ outputs[field] = ', '.join(values) or 'N/A'++ # print(gold[field], '\t\t\t', values)++ return True++ return False+++metricR = CompilationMetric(field='reactions', metric='recall')+from collections import Counter++def reduce_grounded_reactions(grounded_reactions):+ scores = {}+ for score, reaction in grounded_reactions:+ scores[reaction] = scores.get(reaction, 0) + score+ reactions = sorted(scores.items(), key=lambda x: x[1], reverse=True)++ reactions = [r for r, _ in reactions]+ return reactions+++def metricR10(gold, pred, trace=None):+ reactions = pred.reactions++ try:+ reactions = reduce_grounded_reactions(reactions)+ except Exception:+ reactions = Counter(reactions).most_common()+ reactions = [r for r, _ in reactions]++ return metricR(gold, pred.copy(reactions=reactions[:10]), trace)+++def extract_reactions_from_strings(reactions):+ reactions = [r.strip().split('\n')[0].lower().strip('.').strip() for r in reactions]+ reactions = ', '.join(reactions)+ reactions = [r.lower().strip('.').strip() for r in reactions.split(',')]+ reactions = [r for r in reactions if r and r != 'n/a']++ return reactions++class PredictReactions(dspy.Signature):+ __doc__ = f"""Given a snippet from a medical article, identify the adverse drug reactions affecting the patient. If none are mentioned in the snippet, say N/A."""++ title = dspy.InputField()+ context = dspy.InputField()+ reactions = dspy.OutputField(desc="list of comma-separated adverse drug reactions", format=lambda x: ', '.join(x) if isinstance(x, list) else x)++HINT = "Is any of the following reactions discussed in the article snippet? The valid candidates are:"++class GroundedReactionExtractor(dspy.Module):+ def __init__(self, context_window=3000, max_windows=5, num_preds=1):+ super().__init__()++ self.chunk = Chunker(context_window=context_window, max_windows=max_windows)+ self.predict = dspy.ChainOfThoughtWithHint(PredictReactions, n=num_preds)++ def forward(self, title, abstract, context, labels=None):+ hint = f"{HINT} {', '.join(labels.reactions)}." if labels else None+ reactions = []++ for _, snippet in self.chunk(abstract + '\n\n' + context):+ chunk_reactions = self.predict(title=title, context=[snippet], hint=hint)+ reactions.extend(extract_reactions_from_strings(chunk_reactions.completions.reactions))++ reactions = [r for sublist in [ground_v4b(r) for r in reactions] for r in sublist]+ return dspy.Prediction(reactions=reactions)++class BioDexTask(BaseTask):+ def __init__(self):+ global meddra_root++ meddra_root = parse_mdhier(data_dir)+ meddra_root.set_lookup_tables(normalizer=str.lower)++ # Load and configure the datasets.+ dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions")+ official_trainset, official_devset = dataset['train'], dataset['validation']+ trainset, devset = [], []++ for example in tqdm.tqdm(official_trainset):+ if len(trainset) >= 1000: break+ trainset.append(preprocess_example(example))++ for example in tqdm.tqdm(official_devset):+ if len(devset) >= 500: break+ devset.append(preprocess_example(example))++ trainsetX = [dspy.Example(**x).with_inputs('title', 'abstract', 'context', 'labels') for x in trainset]+ trainset = [dspy.Example(**x).with_inputs('title', 'abstract', 'context') for x in trainset]+ devsetX = [dspy.Example(**x).with_inputs('title', 'abstract', 'context', 'labels') for x in devset]+ devset = [dspy.Example(**x).with_inputs('title', 'abstract', 'context') for x in devset]++ self.trainset = trainset+ self.devset = devset++ # Set up metrics+ NUM_THREADS = 16++ self.metric = metricR10++ kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=15)+ self.evaluarteR10 = Evaluate(devset=self.devset, metric=metricR10, **kwargs)++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return GroundedReactionExtractor(context_window=3000, max_windows=5, num_preds=1)++ def get_metric(self):+ return self.metric\ No newline at end of filediff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.pynew file mode 100644index 000000000..6724ebdcd--- /dev/null+++ b/testing/tasks/gsm8k.py@@ -0,0 +1,77 @@+import dspy+from .base_task import BaseTask+from dspy.datasets.gsm8k import gsm8k_metric+import random+import tqdm++from datasets import load_dataset++class CoT(dspy.Module):+ def __init__(self):+ super().__init__()+ self.prog = dspy.ChainOfThought("question -> answer")++ def forward(self, question):+ return self.prog(question=question)++class GSM8KTask(BaseTask):+ def __init__(self):+ dataset = load_dataset("gsm8k", 'main')++ hf_official_train = dataset['train']+ hf_official_test = dataset['test']+ official_train = []+ official_test = []++ for example in tqdm.tqdm(hf_official_train):+ question = example['question']++ answer = example['answer'].strip().split()+ assert answer[-2] == '####'++ gold_reasoning = ' '.join(answer[:-2])+ answer = str(int(answer[-1].replace(',', '')))++ official_train.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))++ for example in tqdm.tqdm(hf_official_test):+ question = example['question']++ answer = example['answer'].strip().split()+ assert answer[-2] == '####'++ gold_reasoning = ' '.join(answer[:-2])+ answer = str(int(answer[-1].replace(',', '')))++ official_test.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))++ rng = random.Random(0)+ rng.shuffle(official_train)++ rng = random.Random(0)+ rng.shuffle(official_test)++ trainset = official_train[:2000]+ devset = official_train[2000:2500]+ testset = official_test[:]++ trainset = [dspy.Example(**x).with_inputs('question') for x in trainset]+ devset = [dspy.Example(**x).with_inputs('question') for x in devset]+ testset = [dspy.Example(**x).with_inputs('question') for x in testset]++ # print(f"Trainset size: {len(trainset)}")+ # print(f"Devset size: {len(devset)}")+ # print(f"Testset size: {len(testset)}")++ self.trainset= trainset+ self.devset = devset+ self.testset = testset+ self.metric = gsm8k_metric++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return CoT()++ def get_metric(self):+ return self.metric\ No newline at end of filediff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.pynew file mode 100644index 000000000..19a5075fc--- /dev/null+++ b/testing/tasks/hotpotqa.py@@ -0,0 +1,49 @@+import dspy+from dspy.evaluate import Evaluate+from .base_task import BaseTask+from dspy.datasets import HotPotQA++class MultiHop(dspy.Module):+ def __init__(self,passages_per_hop):+ super().__init__()+ self.retrieve = dspy.Retrieve(k = passages_per_hop)+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")+ self.generate_answer = dspy.ChainOfThought("context ,question->answer")++ def forward (self,question) :+ context = []+ for hop in range(2):+ query = self.generate_query(context = context, question = question).search_query+ context += self.retrieve(query).passages+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)++class HotPotQATask(BaseTask):+ def __init__(self):+ # Load and configure the datasets.+ hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)+ self.trainset = [x.with_inputs('question') for x in hotpot_dataset.train]+ self.devset = [x.with_inputs('question') for x in hotpot_dataset.dev]++ # Set up metrics+ NUM_THREADS = 16++ metric_EM = dspy.evaluate.answer_exact_match+ self.metric = metric_EM++ def gold_passages_retrieved(example, pred, trace=None):+ gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))+ found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))+ return gold_titles.issubset(found_titles)++ kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=15)+ self.evaluate_EM = Evaluate(devset=self.devset, metric=metric_EM, **kwargs)+ self.evaluate_retrieval = Evaluate(devset=self.devset, metric=gold_passages_retrieved, **kwargs)++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return MultiHop(passages_per_hop=3) #TODO: make it so we can specify # of passages++ def get_metric(self):+ return self.metric+diff --git a/testing/tasks/scone.py b/testing/tasks/scone.pynew file mode 100644index 000000000..e4cbad809--- /dev/null+++ b/testing/tasks/scone.py@@ -0,0 +1,74 @@+import glob+import os+import pandas as pd+import dspy+from .base_task import BaseTask+import random++def load_scone(dirname):+ dfs = []+ for filename in glob.glob(dirname + "/*.csv"):+ df = pd.read_csv(filename, index_col=0)+ df['category'] = os.path.basename(filename).replace(".csv", "")+ dfs.append(df)+ data_df = pd.concat(dfs)++ def as_example(row):+ # The 'one_scoped' file is from an earlier dataset, MoNLI, and+ # so is formatted a bit differently:+ suffix = '' if row['category'] == 'one_scoped' else '_edited'+ # Reformat the hypothesis to be an embedded clause in a question:+ hkey = 'sentence2' + suffix+ question = row[hkey][0].lower() + row[hkey][1: ].strip(".")+ question = f"Can we logically conclude for sure that {question}?"+ # Binary task formulation:+ label = "Yes" if row['gold_label' + suffix] == 'entailment' else "No"+ return dspy.Example({+ "context": row['sentence1' + suffix],+ "question": question,+ "answer": label,+ "category": row['category']+ }).with_inputs("context", "question")++ return list(data_df.apply(as_example, axis=1).values)++class ScoNeSignature(dspy.Signature):+ ("""context, question -> answer""")++ context = dspy.InputField()+ question = dspy.InputField()+ answer = dspy.OutputField(desc="Yes or No")++class ScoNeCoT(dspy.Module):+ def __init__(self):+ super().__init__()+ self.generate_answer = dspy.ChainOfThought(ScoNeSignature)++ def forward(self, context, question):+ return self.generate_answer(context=context, question=question)++class ScoNeTask(BaseTask):+ def __init__(self):+ # !git clone https://github.com/selenashe/ScoNe.git+ if not os.path.exists("ScoNe"):+ os.system("git clone https://github.com/selenashe/ScoNe.git")++ # Load and configure the datasets.+ all_train = load_scone("ScoNe/scone_nli/train")++ random.seed(1)+ random.shuffle(all_train)++ # 1000 random train, 500 random dev:+ self.trainset, self.devset = all_train[: 1000], all_train[1000: 1500]++ metric_EM = dspy.evaluate.answer_exact_match+ self.metric = metric_EM++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return ScoNeCoT()++ def get_metric(self):+ return self.metric\ No newline at end of filediff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.pynew file mode 100644index 000000000..ef3277ed0--- /dev/null+++ b/testing/tasks/tweet.py@@ -0,0 +1,110 @@+import dspy+from dspy.datasets import HotPotQA+from .base_task import BaseTask+from dspy.evaluate import Evaluate+from functools import lru_cache+import openai+from dotenv import load_dotenv+import os++class TweetSignature(dspy.Signature):+ ("""Given context and a question, answer with a tweet""")++ context = dspy.InputField()+ question = dspy.InputField()+ answer = dspy.OutputField(desc="Yes or No")++class TweetCoT(dspy.Module):+ def __init__(self):+ super().__init__()+ self.generate_answer = dspy.ChainOfThought(TweetSignature)++ def forward(self, context, question):+ return self.generate_answer(context=context, question=question)++class MultiHopTweet(dspy.Module):+ def __init__(self,passages_per_hop):+ super().__init__()+ self.retrieve = dspy.Retrieve(k = passages_per_hop)+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")+ self.generate_answer = TweetCoT()++ def forward (self,question) :+ context = []+ for hop in range(2):+ query = self.generate_query(context = context, question = question).search_query+ context += self.retrieve(query).passages+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)++# Define the signature for autoamtic assessments.+class Assess(dspy.Signature):+ """Assess the quality of a tweet along the specified dimension."""++ context = dspy.InputField(desc='ignore if N/A')+ assessed_text = dspy.InputField()+ assessment_question = dspy.InputField()+ assessment_answer = dspy.OutputField(desc="Yes or No")++@lru_cache()+def load_models():+ load_dotenv() # This will load the .env file's variables++ openai.api_key = os.environ.get('OPENAI_API_KEY')+ openai.api_base = os.environ.get('OPENAI_API_BASE')+ gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')+ # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')+ retrieve = dspy.Retrieve(k=5)+ return gpt4T, retrieve++METRIC = None+def metric(gold, pred, trace=None):++ gpt4T, retrieve = load_models()++ question, answer, tweet = gold.question, gold.answer, pred.answer+ context = retrieve(question).passages++ engaging = "Does the assessed text make for a self-contained, engaging tweet?"+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ correct = f"{correct} Does the assessed text above contain the gold answer?"++ with dspy.context(lm=gpt4T):+ faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)+ correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)+ engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)++ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0++ if METRIC is not None:+ if METRIC == 'correct': return correct+ if METRIC == 'engaging': return engaging+ if METRIC == 'faithful': return faithful++ if trace is not None: return score >= 3+ return score / 3.0++class TweetTask(BaseTask):+ def __init__(self):++ # Load the dataset.+ dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)++ # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.+ self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]+ self.devset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]++ self.trainset = [x.with_inputs('question') for x in dataset.train]+ self.devset = [x.with_inputs('question') for x in dataset.dev]++ self.metric = metric++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return MultiHopTweet(passages_per_hop=3) #TODO: make it so we can specify # of passages++ def get_metric(self):+ return self.metric+\ No newline at end of filediff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.pynew file mode 100644index 000000000..2e237d0c2--- /dev/null+++ b/testing/tasks/tweet_metric.py@@ -0,0 +1,148 @@+import dspy+from dspy.datasets import HotPotQA+from .base_task import BaseTask+from dspy.evaluate import Evaluate+from dspy import Example+from functools import lru_cache+import openai+from dotenv import load_dotenv+import os+import uuid+from tqdm import tqdm++class TweetSignature(dspy.Signature):+ ("""Given context and a question, answer with a tweet""")++ context = dspy.InputField()+ question = dspy.InputField()+ answer = dspy.OutputField(desc="Yes or No")++class TweetCoT(dspy.Module):+ def __init__(self):+ super().__init__()+ self.generate_answer = dspy.ChainOfThought(TweetSignature)++ def forward(self, context, question):+ return self.generate_answer(context=context, question=question)++class MultiHopTweet(dspy.Module):+ def __init__(self,passages_per_hop):+ super().__init__()+ self.retrieve = dspy.Retrieve(k = passages_per_hop)+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")+ self.generate_answer = TweetCoT()++ def forward (self,question) :+ context = []+ for hop in range(2):+ query = self.generate_query(context = context, question = question).search_query+ context += self.retrieve(query).passages+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)++# Define the signature for autoamtic assessments.+class Assess(dspy.Signature):+ """Assess the quality of a tweet along the specified dimension."""++ context = dspy.InputField(desc='ignore if N/A')+ assessed_text = dspy.InputField()+ assessment_question = dspy.InputField()+ assessment_answer = dspy.OutputField(desc="Yes or No")++@lru_cache()+def load_models():+ load_dotenv() # This will load the .env file's variables++ openai.api_key = os.environ.get('OPENAI_API_KEY')+ openai.api_base = os.environ.get('OPENAI_API_BASE')+ gpt3T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')+ gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')+ retrieve = dspy.Retrieve(k=5)+ return gpt3T, gpt4T, retrieve++METRIC = None+def metric(gold, pred, trace=None):+ gpt3T, gpt4T, retrieve = load_models()++ question, answer, tweet, context = gold.question, gold.answer, gold.tweet, gold.context+ score_pred = pred.score++ engaging = "Does the assessed text make for a self-contained, engaging tweet?"+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ correct = f"{correct} Does the assessed text above contain the gold answer?"++ with dspy.context(lm=gpt3T): # TODO Update to GPT4+ faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)+ correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)+ engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)++ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0++ return 1 - abs(score - score_pred) # We want a score we can maximize, so take the negative L1 norm and add 1++class TweetMetric(dspy.Module):+ def __init__(self):+ super().__init__()+ self.engaging = dspy.Predict(Assess)+ self.faithful = dspy.Predict(Assess)+ self.correct = dspy.Predict(Assess)++ def forward (self, tweet, context, question, answer) :+ engaging = "Does the assessed text make for a self-contained, engaging tweet?"+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."+ correct = f"{correct} Does the assessed text above contain the gold answer?"++ faithful = self.faithful(context=context, assessed_text=tweet, assessment_question=faithful)+ correct = self.correct(context='N/A', assessed_text=tweet, assessment_question=correct)+ engaging = self.engaging(context='N/A', assessed_text=tweet, assessment_question=engaging)++ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0++ return dspy.Prediction(score= score/3.0)++class TweetMetricTask(BaseTask):+ def __init__(self):++ # Load the dataset.+ dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)++ # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.+ trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]+ devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]+ self.trainset = []+ self.devset = []++ gpt3T, gpt4T, retrieve = load_models()++ with dspy.context(lm=gpt3T):+ for ex in tqdm(trainset_temp, desc="Preprocessing Trainset"):+ context = retrieve(ex.question).passages+ question = ex.question+ answer = ex.answer+ tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer+ example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}++ self.trainset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='train').with_inputs('context', 'question', 'answer', 'tweet'))++ for ex in tqdm(devset_temp, desc="Preprocessing Devset"):+ context = retrieve(ex.question).passages+ question = ex.question+ answer = ex.answer+ tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer+ example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}++ self.devset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))++ self.metric = metric++ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)++ def get_program(self):+ return TweetMetric()++ def get_metric(self):+ return self.metric+\ No newline at end of file