Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Added optimizer testing framework

Michael Ryan285eebcd9494
diff --git a/testing/README.md b/testing/README.md
new file mode 100644
index 000000000..60ed5025e
--- /dev/null
+++ b/testing/README.md
@@ -0,0 +1,91 @@
+# Optimizer Tester
+
+Optimizer Tester is intended to allow simple and repeatable testing of DSPy Optimizers. This is a development tool for the creation of more optimizers, and the validation that they work across tasks. This is still under development and subject to change.
+
+## Usage
+
+To use the Optimizer Tester in code instantiate an OptimizerTester object:
+
+```python
+from testing.optimizer_tester import OptimizerTester
+
+tester = OptimizerTester()
+```
+
+The default verison (no parameters) expects a llama model hosted on ports [7140, 7141, 7142, 7143] and OpenAI keys stored in a .env file (OPENAI_API_KEY and OPENAI_API_BASE).
+
+If you prefer to specify your own model parameters then you can pass models into the OptimizerTester
+
+```python
+task_model = dspy.HFClientTGI(...)
+prompt_model = dspy.OpenAI(...)
+
+tester = OptimizerTester()
+```
+
+If you just want to get baseline results for a particular task you're ready to go!
+
+```python
+tester.test_baseline(datasets=["HotPotQA", "GSM8K", "Scone"])
+```
+
+If you want to test out a custom optimizer you'll have to write a quick function to call it properly:
+
+```python
+def your_optimizer_caller(default_program, trainset, devset, test_name, dataset_name, kwargs):
+ # initialize your teleprompter (optimizer) here!
+ teleprompter = my_custom_optimizer(metric=kwargs["metric"], ...)
+
+ # call your optimizer on the default_program here!
+ compiled_program = teleprompter.compile(default_program.deepcopy(), trainset=trainset, ...)
+
+ # if you wish to tweak any of the outputs to the csv file you can do that here
+ output = {
+ "test_name": "my_optimizer-" + dataset_name + "-" + test_name
+ }
+
+ # return the compiled program and modified output (or empty dict if no changes made)
+ return compiled_program, output
+```
+
+You can use these kwargs:
+- breadth=self.BREADTH
+- depth=self.DEPTH
+- temperature=self.TEMPERATURE
+- prompt_model=self.prompt_model
+- view_data=False
+- log_dir=log_dir
+- metric=task.get_metric()
+- task_model=self.task_model
+
+These are somewhat specific to the testing for the signature optimizer teleprompter which this tool was originally built for, so you probably will not use most of these.
+
+Here is an example of an implemented version of the function:
+
+```python
+def signature_optimizer_default(default_program, trainset, devset, test_name, dataset_name, kwargs):
+ eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)
+
+ teleprompter = SignatureOptimizer(prompt_model=kwargs["prompt_model"], task_model=kwargs["task_model"], metric=kwargs["metric"], breadth=kwargs["breadth"], depth=kwargs["depth"], init_temperature=kwargs["temperature"], verbose=False, log_dir=kwargs["log_dir"])
+ compiled_program = teleprompter.compile(default_program.deepcopy(), devset=trainset, evalset=devset, eval_kwargs=eval_kwargs)
+
+ output = {
+ "meta_prompt_style": "best",
+ "view_data": True,
+ "test_name": dataset_name + "_" + test_name
+ }
+
+ return compiled_program, output
+```
+
+Then you can use the 'test_optimizer_default' test to run the tests on your optimizer:
+
+```python
+tester.test_optimizer_default(signature_optimizer_default, datasets=["BioDex", "Tweet", "Tweet Metric"])
+```
+
+We are working on more optimizer tests with interesting compositions of optimizers, and other hyperparameter adjustments, which will be added as they are developed.
+
+## Important Note
+
+The BioDex Task requires an external download. Please see the `/tasks/biodex.py` file for a link to download details and a note about where to insert the download path in the code.
\ No newline at end of file
diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.py
new file mode 100644
index 000000000..ffcf3b262
--- /dev/null
+++ b/testing/optimizer_tester.py
@@ -0,0 +1,204 @@
+from .tasks.scone import ScoNeTask
+from .tasks.hotpotqa import HotPotQATask
+from .tasks.gsm8k import GSM8KTask
+from .tasks.biodex import BioDexTask
+from .tasks.tweet import TweetTask
+from .tasks.tweet_metric import TweetMetricTask
+from dspy.evaluate import Evaluate
+from dotenv import load_dotenv
+import openai
+import os
+import dspy
+import csv
+import datetime
+from timeit import default_timer as timer
+
+datasets = ["ScoNe", "HotPotQA", "GSM8K", "BioDex", "Tweet"]
+
+class OptimizerTester:
+ def __init__(self, datasets=datasets, default_train_num = 200, default_dev_num = 100, default_test_num = 200,
+ num_threads = 10, default_breadth = 10, default_depth = 3, default_temperature = 1.4,
+ prompt_model_name = "gpt-3.5-turbo-1106", task_model_name = "meta-llama/Llama-2-13b-chat-hf",
+ prompt_model = None, task_model = None,
+ colbert_v2_endpoint = "http://20.102.90.50:2017/wiki17_abstracts"):
+ self.datasets = datasets
+ self.TRAIN_NUM = default_train_num
+ self.DEV_NUM = default_dev_num
+ self.EVAL_NUM = default_test_num
+ self.NUM_THREADS = num_threads
+ self.BREADTH = default_breadth
+ self.DEPTH = default_depth
+ self.TEMPERATURE = default_temperature
+ self.PROMPT_MODEL_NAME = prompt_model_name
+ self.TASK_MODEL_NAME = task_model_name
+ self.COLBERT_V2_ENDPOINT = colbert_v2_endpoint
+
+ load_dotenv() # This will load the .env file's variables
+
+ openai.api_key = os.environ.get('OPENAI_API_KEY')
+ openai.api_base = os.environ.get('OPENAI_API_BASE')
+
+ # Prompt gen model
+ if not prompt_model:
+ self.prompt_model = dspy.OpenAI(model=self.PROMPT_MODEL_NAME, max_tokens=150)
+ else:
+ self.prompt_model = prompt_model
+
+ # Task model
+ if not task_model:
+ self.task_model = dspy.HFClientTGI(model=self.TASK_MODEL_NAME, port=[7140, 7141, 7142, 7143], max_tokens=150)
+ else:
+ self.task_model = task_model
+ self.colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)
+
+ dspy.settings.configure(rm=self.colbertv2, lm=self.task_model)
+
+ def write_to_csv(self, folder_name, file_name, data):
+ # Ensure the output directory exists
+ os.makedirs(folder_name, exist_ok=True)
+ file_path = os.path.join(folder_name, file_name)
+
+ # Check if file exists to determine if headers should be written
+ file_exists = os.path.isfile(file_path)
+
+ with open(file_path, mode='a', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+
+ headers = ['test_name', 'train_score', 'dev_score', 'eval_score', 'run_time (sec)', 'train_size', 'dev_size', 'eval_size', 'task_name', 'signature_optimized', 'prompt_model_name', 'task_model_name', 'breadth', 'depth', 'meta_prompt_style', 'fewshot_before', 'fewshot_after', 'temperature', 'fewshot_candidates_num', 'max_bootstrapped_demos', 'bootstrapping', 'view_data', 'optimizer_log_dir', 'additional_notes', 'misc']
+
+ # Write headers if the file is being created
+ if not file_exists:
+ writer.writerow(headers)
+
+ # Format the data
+ formatted_data = ['NA'] * len(headers)
+ formatted_data[-1] = ""
+ for key in data.keys():
+ if key in headers:
+ formatted_data[headers.index(key)] = data[key]
+ else:
+ formatted_data[-1] += f"{key}: {data[key]}\n"
+ # Write the data
+ writer.writerow(formatted_data)
+
+
+ def load_dataset(self,dataset):
+ ds = None
+ dataset = dataset.lower()
+ if dataset == "scone":
+ ds = ScoNeTask()
+ elif dataset == "hotpotqa":
+ ds = HotPotQATask()
+ elif dataset == "gsm8k":
+ ds = GSM8KTask()
+ elif dataset == "biodex":
+ ds = BioDexTask()
+ elif dataset == "tweet":
+ ds = TweetTask()
+ elif dataset == "tweet_metric":
+ ds = TweetMetricTask()
+ else:
+ raise ValueError("Invalid dataset name.")
+ ds.set_splits(TRAIN_NUM=self.TRAIN_NUM, DEV_NUM=self.DEV_NUM, EVAL_NUM=self.EVAL_NUM)
+ return ds
+
+ # Computes baseline results for a given dataset
+ def test_baseline(self, datasets=datasets, test_name="baseline"):
+ for dataset in datasets:
+ task = self.load_dataset(dataset)
+ evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ evaluate_eval = Evaluate(devset=task.get_evalset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ default_program = task.get_program()
+
+ # Evaluate the default program
+ default_results_train = evaluate_train(default_program)
+ default_results_dev = evaluate_dev(default_program)
+ default_results_eval = evaluate_eval(default_program)
+
+ # Write the results to a csv
+ self.write_to_csv('outputs', 'results.csv', {
+ 'test_name': dataset + "_" + test_name,
+ 'train_score': default_results_train,
+ 'dev_score': default_results_dev,
+ 'eval_score': default_results_eval,
+ 'run_time (sec)': 0,
+ 'train_size': len(task.get_trainset()),
+ 'dev_size': len(task.get_devset()),
+ 'eval_size': len(task.get_evalset()),
+ 'task_name': dataset,
+ 'signature_optimized': False,
+ 'prompt_model_name': self.PROMPT_MODEL_NAME,
+ 'task_model_name': self.TASK_MODEL_NAME,
+ 'breadth': 'NA',
+ 'depth': 'NA',
+ 'meta_prompt_style': 'default',
+ 'fewshot_before': False,
+ 'fewshot_after': False,
+ 'temperature': self.TEMPERATURE,
+ 'fewshot_candidates_num': 0,
+ 'max_bootstrapped_demos': 0,
+ 'bootstrapping': False,
+ 'view_data': False,
+ 'optimizer_log_dir': 'NA',
+ 'additional_notes': '',
+ 'misc': ''
+ })
+
+ def test_optimizer_default(self, optimizer_function, datasets=datasets, test_name="default"):
+
+ for dataset in datasets:
+ task = self.load_dataset(dataset)
+ evaluate_train = Evaluate(devset=task.get_trainset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ evaluate_dev = Evaluate(devset=task.get_devset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ evaluate_eval = Evaluate(devset=task.get_evalset(), metric=task.get_metric(), num_threads=self.NUM_THREADS, display_progress=True)
+ default_program = task.get_program()
+
+ # Set up the optimizer kwargs
+ date_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+ log_dir = "log_dir/" + dataset + "_" + test_name + "_" + date_timestamp + "/"
+ os.makedirs(log_dir, exist_ok=True)
+ kwargs = dict(breadth=self.BREADTH, depth=self.DEPTH, temperature=self.TEMPERATURE, prompt_model=self.prompt_model, view_data=False, log_dir=log_dir, metric=task.get_metric(), task_model=self.task_model)
+
+ # Optimize the default program
+ start = timer()
+ optimized_program, output_dict = optimizer_function(default_program, task.get_trainset(), task.get_devset(), test_name, dataset, kwargs)
+ end = timer()
+
+ # Evaluate the optimized program
+ optimized_results_train = evaluate_train(optimized_program)
+ optimized_results_dev = evaluate_dev(optimized_program)
+ optimized_results_eval = evaluate_eval(optimized_program)
+
+ output = {
+ 'test_name': dataset + "_" + test_name,
+ 'train_score': optimized_results_train,
+ 'dev_score': optimized_results_dev,
+ 'eval_score': optimized_results_eval,
+ 'run_time (sec)': end - start,
+ 'train_size': len(task.get_trainset()),
+ 'dev_size': len(task.get_devset()),
+ 'eval_size': len(task.get_evalset()),
+ 'task_name': dataset,
+ 'signature_optimized': True,
+ 'prompt_model_name': self.PROMPT_MODEL_NAME,
+ 'task_model_name': self.TASK_MODEL_NAME,
+ 'breadth': self.BREADTH,
+ 'depth': self.DEPTH,
+ 'meta_prompt_style': 'default',
+ 'fewshot_before': False,
+ 'fewshot_after': False,
+ 'temperature': self.TEMPERATURE,
+ 'fewshot_candidates_num': 0,
+ 'max_bootstrapped_demos': 0,
+ 'bootstrapping': False,
+ 'view_data': False,
+ 'optimizer_log_dir': log_dir,
+ 'additional_notes': '',
+ 'misc': ''
+ }
+
+ output.update(output_dict)
+
+ # Write the results to a csv
+ self.write_to_csv('outputs', 'results.csv', output)
\ No newline at end of file
diff --git a/testing/tasks/__init__.py b/testing/tasks/__init__.py
new file mode 100644
index 000000000..f5034e36f
--- /dev/null
+++ b/testing/tasks/__init__.py
@@ -0,0 +1,7 @@
+from .base_task import BaseTask
+from .gsm8k import GSM8KTask
+from .hotpotqa import HotPotQATask
+from .scone import ScoNeTask
+from .biodex import BioDexTask
+from .tweet import TweetTask
+from .tweet_metric import TweetMetricTask
\ No newline at end of file
diff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.py
new file mode 100644
index 000000000..c7f8f42b6
--- /dev/null
+++ b/testing/tasks/base_task.py
@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+
+class BaseTask(ABC):
+ def __init__(self):
+ pass
+
+ @abstractmethod
+ def get_program(self):
+ pass
+
+ @abstractmethod
+ def get_metric(self):
+ pass
+
+ def get_trainset(self, TRAIN_NUM=None):
+ if TRAIN_NUM:
+ self.TRAIN_NUM = TRAIN_NUM
+ return self.trainset[:TRAIN_NUM]
+ else:
+ return self.trainset[:self.TRAIN_NUM]
+
+ def get_devset(self, TRAIN_NUM=None, DEV_NUM=None):
+ if TRAIN_NUM:
+ self.TRAIN_NUM = TRAIN_NUM
+ if DEV_NUM:
+ self.DEV_NUM = DEV_NUM
+
+ index = -1
+ if DEV_NUM and TRAIN_NUM:
+ index = max(len(self.trainset) - DEV_NUM, TRAIN_NUM)
+ else:
+ index = max(len(self.trainset) - self.DEV_NUM, self.TRAIN_NUM)
+ return self.trainset[index:]
+
+ def get_evalset(self, EVAL_NUM=None):
+ if EVAL_NUM:
+ self.EVAL_NUM = EVAL_NUM
+ return self.devset[:EVAL_NUM]
+ else:
+ return self.devset[:self.EVAL_NUM]
+
+ def set_splits(self, TRAIN_NUM=None, DEV_NUM=None, EVAL_NUM=None):
+ if TRAIN_NUM:
+ self.TRAIN_NUM = TRAIN_NUM
+ if DEV_NUM:
+ self.DEV_NUM = DEV_NUM
+ if EVAL_NUM:
+ self.EVAL_NUM = EVAL_NUM
\ No newline at end of file
diff --git a/testing/tasks/biodex.py b/testing/tasks/biodex.py
new file mode 100644
index 000000000..43b2d2e4a
--- /dev/null
+++ b/testing/tasks/biodex.py
@@ -0,0 +1,633 @@
+from __future__ import annotations
+from .base_task import BaseTask
+import sys
+import dspy
+from dspy.evaluate import Evaluate
+from dsp.utils import deduplicate
+import tqdm
+import datasets
+import sys
+import math
+from functools import lru_cache
+import os
+import re
+import pickle
+from pydantic import BaseModel
+from rapidfuzz import process
+from typing import Union, Optional, Dict, DefaultDict, List
+from enum import Enum
+from collections import defaultdict
+
+# Must point to a MedDra download with mdhier.asc
+data_dir = '/future/u/okhattab/data/2023/MedDraV2/meddra_23_0_english/MedAscii' # NOTE: EDIT THIS LINE
+
+global meddra_root
+meddra_root = None
+
+def _remove_non_llt(candidates):
+ return [c for c in candidates if c.level == Level.LLT]
+
+
+def get_path(node):
+ path = f"under {node.parent.term} < {node.parent.parent.term} < {node.parent.parent.parent.term}"
+ return path
+
+
+def flatten(ll):
+ return [x for l in ll for x in l]
+
+
+def _matches_to_terms_and_paths(matches):
+ s = []
+ for n in matches:
+ new_s = f"{n.node.term} ({n.score} score for query '{n.query}'; {n.node.count} counts; {get_path(n.node)})"
+ s.append(new_s)
+ return s
+
+
+class Level(Enum):
+ ROOT = 0
+ SOC = 1
+ HLTG = 2
+ HLT = 3
+ LLT = 4
+
+ def __add__(self, other: int):
+ if self.ROOT.value <= self.value + other <= self.LLT.value:
+ return Level(self.value + other)
+ else:
+ return None
+
+ def __lt__(self, other) -> bool:
+ return self.value < other.value
+
+ def __le__(self, other) -> bool:
+ return self.value <= other.value
+
+ def __gt__(self, other) -> bool:
+ return self.value > other.value
+
+ def __ge__(self, other) -> bool:
+ return self.value >= other.value
+
+ def __sub__(self, other) -> int:
+ return self.value - other.value
+
+
+class Node(BaseModel):
+ id: str
+ term: str
+ level: Level
+ count: int = 0
+
+ parent: 'Node' = None
+ children: Dict[str, 'Node'] = {}
+
+ # maps for efficient nested children lookups
+ # typically only used for the root node
+ id_to_nodes: Optional[DefaultDict[str, List['Node']]] = None
+ term_to_nodes: Optional[DefaultDict[str, List['Node']]] = None
+
+ # text normalizer to build lookup tables
+ # typically only used for the root node
+ normalizer: function = lambda x: x
+
+ def get_parent_at_level(self, target_level: Level) -> Union['Node', None]:
+ # check if current node is deeper than target_level
+ level_diff = target_level - self.level
+ if level_diff > 0:
+ return None
+ else:
+ node = self
+ for _ in range(-level_diff):
+ node = node.parent
+ return node
+
+ def get_children_at_level(self, target_level: Level) -> List['Node']:
+ raise NotImplementedError
+
+ def lookup_id(self, id: str) -> Union[List['Node'], None]:
+ # create lookup tables if they do not exists
+ if self.id_to_nodes == None:
+ self.set_lookup_tables()
+ # lookup
+
+ if id in self.id_to_nodes:
+ return self.id_to_nodes[id]
+ else:
+ return None
+
+ def lookup_term(self, term: str) -> Union[List['Node'], None]:
+ # create lookup tables if they do not exists
+ if self.term_to_nodes == None:
+ raise RuntimeError("First set the lookup tables using 'set_lookup_tables'.")
+ # lookup
+ term = self.normalizer(term)
+ if term in self.term_to_nodes:
+ return self.term_to_nodes[term]
+ else:
+ return None
+
+ def match(
+ self,
+ query: str,
+ fuzzy_limit: int = 20,
+ fuzzy_score_cutoff: int = 80,
+ return_nodes: bool = False,
+ ):
+ """Return a tuple containing exact match and best fuzzy matches of LLT terms in ontology.
+
+ Args:
+ query (str): String being matched into ontology
+ fuzzy_limit (int, optional): Maximal limit of fuzzy matches to return. Defaults to 20.
+ fuzzy_score_cutoff (int, optional): Minimal fuzzy matching score, between 0 and 100. Defaults to 80.
+ return_nodes (bool, optional): If true, return Node objects instead of paths
+
+ Returns:
+ tuple: Tuple(exact match or None, list of fuzzy matches)
+ """
+ exact_candidate = []
+ fuzzy_candidates = []
+
+ # exact match
+ exact_matches = self.lookup_term(query)
+ if exact_matches is not None:
+ exact_candidate = _remove_non_llt(exact_matches)
+ exact_candidate = [
+ Match(query=query, score=100, node=node) for node in exact_candidate
+ ]
+
+ # fuzzy match
+ fuzzy_terms = process.extract(
+ query,
+ self.term_to_nodes.keys(),
+ limit=fuzzy_limit,
+ score_cutoff=fuzzy_score_cutoff,
+ )
+
+ fuzzy_matches = []
+ for term, sim, _ in fuzzy_terms:
+ applicable_nodes = _remove_non_llt(self.term_to_nodes[term])
+ for n in applicable_nodes:
+ fuzzy_matches.append(Match(node=n, score=sim, query=query))
+
+ if return_nodes:
+ return exact_candidate, fuzzy_matches
+ else:
+ exact_term = (
+ _matches_to_terms_and_paths(exact_candidate) if exact_candidate else []
+ )
+ fuzzy_terms = _matches_to_terms_and_paths(fuzzy_matches)
+ return exact_term, fuzzy_terms
+
+ def set_lookup_tables(self, normalizer: function = lambda x: x) -> None:
+ # set normalizer
+ self.normalizer = normalizer
+
+ # get a flat list of all nodes
+ self.term_to_nodes = defaultdict(list)
+ self.id_to_nodes = defaultdict(list)
+
+ # breadth-first traversal
+ candidates = [self]
+ while len(candidates):
+ candidate = candidates.pop(0)
+ # track nodes
+ self.term_to_nodes[self.normalizer(candidate.term)].append(candidate)
+ self.id_to_nodes[candidate.id].append(candidate)
+ # add candidate's children to candidate list
+ if candidate.children:
+ candidates.extend(candidate.children.values())
+ return None
+
+ def is_equivalent_node(self, other: 'Node') -> bool:
+ # return true if two nodes are the same, siblings or in a parent-child relation
+ return (
+ (self == other)
+ or (self.parent == other)
+ or (self == other.parent)
+ or (self.parent == other.parent)
+ )
+
+ def terms_equivalent(self, term1: str, term2: str) -> Union[bool, None]:
+ # lookup both terms
+ term1_nodes = self.lookup_term(term1)
+ term2_nodes = self.lookup_term(term2)
+ # if at least one term not found
+ if not term1_nodes or not term2_nodes:
+ return None
+ # compare all nodes
+ for node1 in term1_nodes:
+ for node2 in term2_nodes:
+ if node1.is_equivalent_node(node2):
+ return True
+ return False
+
+ def __key(self):
+ return (self.id, self.term, self.parent, len(self.children))
+
+ def __hash__(self):
+ return hash(self.__key())
+
+ def __eq__(self, other: 'Node') -> bool:
+ # undeep equals operator, does not check children
+ return self.__key() == other.__key()
+
+ def __str__(self) -> str:
+ return f"Node(id={self.id}, term={self.term}, level={self.level}, parent={self.parent.term if self.parent else None}, #children={len(self.children)}, count={self.count})"
+
+ def __repr__(self) -> str:
+ return self.__str__()
+
+ def get_path(self):
+ path = f"under {self.parent.term} < {self.parent.parent.term} < {self.parent.parent.parent.term}"
+ return path
+
+ @property
+ def all_nodes(self):
+ return flatten(list(self.id_to_nodes.values()))
+
+ @property
+ def all_llt_nodes(self):
+ return _remove_non_llt(self.all_nodes)
+
+
+class Match(BaseModel):
+ node: Node
+ score: int
+ query: str
+
+def _create_child_if_not_exists(
+ parent: Node, child_id: str, child_term: str, term_to_count: dict
+):
+ parent_level = parent.level
+ # get child level if still appropriate
+ child_level = parent_level + 1
+ if child_level is None:
+ return None
+ # get child if exists
+ if child_id in parent.children:
+ child = parent.children[child_id]
+ # create child if not exists
+ else:
+ child_term_dict = child_term.lower().replace("'", "^")
+ if child_term_dict in term_to_count:
+ count = term_to_count[child_term_dict]
+ else:
+ count = 0
+ child = Node(
+ id=child_id, term=child_term, level=child_level, parent=parent, count=count
+ )
+ parent.children.update({child_id: child})
+ return child
+
+def parse_mdhier(data_dir):
+ file = os.path.join(data_dir, "mdhier.asc")
+ if not os.path.isfile(file):
+ raise FileNotFoundError(f"File {file} not found.")
+ count_file = os.path.join(data_dir, "priors.pickle")
+ if not os.path.isfile(count_file):
+ raise FileNotFoundError(f"File {count_file} not found.")
+
+ # get regex
+ id_regex = r"\d{8}"
+ term_regex = r"[^$]*"
+ regex = rf"({id_regex})\$({id_regex})\$({id_regex})\$({id_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$({term_regex})\$\$({id_regex})\$[YN]\$"
+ regex = re.compile(regex)
+
+ # get data
+ lines = []
+ with open(file, "r") as fp:
+ lines = fp.readlines()
+ text = "\n".join(lines)
+
+ matches = regex.findall(text)
+
+ print(f"Read {len(lines)} lines.")
+ if len(lines) != len(matches):
+ raise RuntimeError(
+ f"Error parsing mdhier.asc. Expected number of matches to match number of lines."
+ )
+
+ # load counts
+ with open(count_file, "rb") as fp:
+ counts = pickle.load(fp)
+
+ # create ontology tree
+ root = Node(id="root", term="root", level=Level.ROOT)
+ for match in matches:
+ (
+ llt_id,
+ hlt_id,
+ hltg_id,
+ soc_id,
+ llt_term,
+ hlt_term,
+ hltg_term,
+ soc_term_1,
+ soc_term_2,
+ _,
+ ) = match
+ # recursivelty add the nodes specified by this line
+ soc_node = _create_child_if_not_exists(root, soc_id, soc_term_1, counts)
+ hltg_node = _create_child_if_not_exists(soc_node, hltg_id, hltg_term, counts)
+ hlt_node = _create_child_if_not_exists(hltg_node, hlt_id, hlt_term, counts)
+ llt_node = _create_child_if_not_exists(hlt_node, llt_id, llt_term, counts)
+
+ # print(llt_node.parent == hlt_node)
+ # print(hlt_node.parent == hltg_node)
+ # print(hltg_node.parent == soc_node)
+ # print(soc_node.parent == root)
+
+ # print(Level.SOC < Level.LLT)
+ # print(Level.SOC > Level.LLT)
+ # print(Level.SOC - Level.LLT)
+
+ # print(soc_node.get_parent_at_level(Level.ROOT))
+ # print(soc_node.get_parent_at_level(Level.SOC))
+ # print(soc_node.get_parent_at_level(Level.LLT))
+ # print(llt_node.get_parent_at_level(Level.SOC))
+
+ # print(root.lookup_term("COVID-19 pneumonia"))
+ # print(root.lookup_term("COVID-19 pneumonia")[0].parent)
+ return root
+
+@lru_cache(maxsize=100000)
+def base_ground(reaction, K):
+ global meddra_root
+ kwargs = dict(fuzzy_limit=K, fuzzy_score_cutoff=75, return_nodes=True)
+ exact_matches, fuzzy_matches = meddra_root.match(reaction, **kwargs)
+
+ return exact_matches, fuzzy_matches
+
+
+@lru_cache(maxsize=100000)
+def ground_v1(reaction, K=3):
+ """
+ Prefers exact matches over fuzzy matches, when available.
+ """
+
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = exact_matches[:1] or fuzzy_matches
+ matches = [(match.score / 100.0, match.node.term) for match in matches]
+ matches = sorted(list(set(matches)), reverse=True)
+
+ return tuple(matches)
+
+@lru_cache(maxsize=100000)
+def ground_v2(reaction, K=1):
+ """
+ When K=1 (default), returns exact matches (if available) or the best fuzzy match.
+ """
+
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = [(match.score / 100.0, match.node.term) for match in (exact_matches[:1] + fuzzy_matches)]
+ matches = sorted(list(set(matches)), reverse=True)[:K]
+
+ return tuple(matches)
+
+# Returns the best three matches (including one exact match, if available).
+ground_v3 = lambda reaction: ground_v2(reaction, K=3)
+
+@lru_cache(maxsize=100000)
+def ground_v4(reaction, K=3):
+ """
+ Returns the best three matches (including one exact match, if available) and applies a prior.
+ """
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = [((match.score / 100.0) * math.sqrt(match.node.count + 0.1), match.node.term)
+ for match in (exact_matches[:1] + fuzzy_matches)]
+ matches = sorted(list(set(matches)), reverse=True)[:K]
+
+ return tuple(matches)
+
+@lru_cache(maxsize=100000)
+def ground_v4b(reaction, K=3):
+ """
+ Returns the best three matches (including one exact match, if available) and applies a prior.
+ """
+
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = [((match.score / 100.0) * math.log(match.node.count + 0.1), match.node.term)
+ for match in (exact_matches[:1] + fuzzy_matches)]
+ matches = sorted(list(set(matches)), reverse=True)[:K]
+
+ return tuple(matches)
+
+@lru_cache(maxsize=100000)
+def ground_v4c(reaction, K=3):
+ """
+ Returns the best three matches (including one exact match, if available) and applies a prior.
+ """
+
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = [((match.score / 100.0) * (match.node.count + 0.1), match.node.term)
+ for match in (exact_matches[:1] + fuzzy_matches)]
+ matches = sorted(list(set(matches)), reverse=True)[:K]
+
+ return tuple(matches)
+
+
+@lru_cache(maxsize=100000)
+def ground_v5(reaction, K=3):
+ exact_matches, fuzzy_matches = base_ground(reaction, K)
+
+ matches = [((match.score / 100.0)**2 * math.log(match.node.count + 0.1), match.node.term)
+ for match in (exact_matches[:1] + fuzzy_matches)]
+ matches = sorted(list(set(matches)), reverse=True)[:K]
+
+ return tuple(matches)
+
+def preprocess_example(example):
+ title = example['title']
+ abstract = example['abstract']
+ context = example['fulltext_processed'].split('\n\nTEXT:\n', 1)[-1]
+ reactions = [r.strip().lower() for r in example['reactions'].split(',')]
+
+ example = dict(title=title, abstract=abstract, context=context, reactions=reactions)
+ example['labels'] = dspy.Example(reactions=reactions)
+
+ return example
+
+class Chunker:
+ def __init__(self, context_window=3000, max_windows=5):
+ self.context_window = context_window
+ self.max_windows = max_windows
+ self.window_overlap = 0.02
+
+ def __call__(self, paper):
+ snippet_idx = 0
+
+ while snippet_idx < self.max_windows and paper:
+ endpos = int(self.context_window * (1.0 + self.window_overlap))
+ snippet, paper = paper[:endpos], paper[endpos:]
+
+ next_newline_pos = snippet.rfind('\n')
+ if paper and next_newline_pos != -1 and next_newline_pos >= self.context_window // 2:
+ paper = snippet[next_newline_pos+1:] + paper
+ snippet = snippet[:next_newline_pos]
+
+ yield snippet_idx, snippet.strip()
+ snippet_idx += 1
+
+def get_set_precision_and_recalls(l1, l2):
+ # lower case and strip
+ l1 = [x.strip().lower() for x in l1]
+ l2 = [x.strip().lower() for x in l2]
+
+ s1, s2 = set(l1), set(l2)
+ intersect = s1.intersection(s2)
+
+ p, r = len(intersect) / max(1, len(s1)), len(intersect) / max(1, len(s2))
+ f1 = 2 * (p * r) / max((p + r), 1)
+
+ return dspy.Prediction(precision=p, recall=r, f1=f1)
+
+def finegrained_metrics(gold, pred):
+ reaction_metrics = get_set_precision_and_recalls(pred.reactions, gold.reactions)
+ return reaction_metrics
+
+
+class CompilationMetric:
+ __name__ = 'compilation_metric'
+
+ def __init__(self, field='reactions', metric='recall', perfect_precision=False):
+ self.field = field
+ self.metric = metric
+ self.perfect_precision = perfect_precision
+
+ def __call__(self, gold, pred, trace=None):
+ field = self.field
+ metric = finegrained_metrics(gold, pred) #[field]
+
+ if trace is None: # eval
+ return metric[self.metric]
+
+ if metric.precision >= 0.5 and metric.recall >= 0.5:
+ for trace_idx in range(len(trace)):
+ *_, outputs = trace[trace_idx]
+
+ values = outputs[field].split(', ')
+ values = [r.lower().strip('.').strip() for r in values]
+ values = deduplicate([r for r in values if r and r != 'n/a'])
+
+ if self.perfect_precision:
+ values = [r for r in values if r in gold[self.field]]
+
+ outputs[field] = ', '.join(values) or 'N/A'
+
+ # print(gold[field], '\t\t\t', values)
+
+ return True
+
+ return False
+
+
+metricR = CompilationMetric(field='reactions', metric='recall')
+from collections import Counter
+
+def reduce_grounded_reactions(grounded_reactions):
+ scores = {}
+ for score, reaction in grounded_reactions:
+ scores[reaction] = scores.get(reaction, 0) + score
+ reactions = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+
+ reactions = [r for r, _ in reactions]
+ return reactions
+
+
+def metricR10(gold, pred, trace=None):
+ reactions = pred.reactions
+
+ try:
+ reactions = reduce_grounded_reactions(reactions)
+ except Exception:
+ reactions = Counter(reactions).most_common()
+ reactions = [r for r, _ in reactions]
+
+ return metricR(gold, pred.copy(reactions=reactions[:10]), trace)
+
+
+def extract_reactions_from_strings(reactions):
+ reactions = [r.strip().split('\n')[0].lower().strip('.').strip() for r in reactions]
+ reactions = ', '.join(reactions)
+ reactions = [r.lower().strip('.').strip() for r in reactions.split(',')]
+ reactions = [r for r in reactions if r and r != 'n/a']
+
+ return reactions
+
+class PredictReactions(dspy.Signature):
+ __doc__ = f"""Given a snippet from a medical article, identify the adverse drug reactions affecting the patient. If none are mentioned in the snippet, say N/A."""
+
+ title = dspy.InputField()
+ context = dspy.InputField()
+ reactions = dspy.OutputField(desc="list of comma-separated adverse drug reactions", format=lambda x: ', '.join(x) if isinstance(x, list) else x)
+
+HINT = "Is any of the following reactions discussed in the article snippet? The valid candidates are:"
+
+class GroundedReactionExtractor(dspy.Module):
+ def __init__(self, context_window=3000, max_windows=5, num_preds=1):
+ super().__init__()
+
+ self.chunk = Chunker(context_window=context_window, max_windows=max_windows)
+ self.predict = dspy.ChainOfThoughtWithHint(PredictReactions, n=num_preds)
+
+ def forward(self, title, abstract, context, labels=None):
+ hint = f"{HINT} {', '.join(labels.reactions)}." if labels else None
+ reactions = []
+
+ for _, snippet in self.chunk(abstract + '\n\n' + context):
+ chunk_reactions = self.predict(title=title, context=[snippet], hint=hint)
+ reactions.extend(extract_reactions_from_strings(chunk_reactions.completions.reactions))
+
+ reactions = [r for sublist in [ground_v4b(r) for r in reactions] for r in sublist]
+ return dspy.Prediction(reactions=reactions)
+
+class BioDexTask(BaseTask):
+ def __init__(self):
+ global meddra_root
+
+ meddra_root = parse_mdhier(data_dir)
+ meddra_root.set_lookup_tables(normalizer=str.lower)
+
+ # Load and configure the datasets.
+ dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions")
+ official_trainset, official_devset = dataset['train'], dataset['validation']
+ trainset, devset = [], []
+
+ for example in tqdm.tqdm(official_trainset):
+ if len(trainset) >= 1000: break
+ trainset.append(preprocess_example(example))
+
+ for example in tqdm.tqdm(official_devset):
+ if len(devset) >= 500: break
+ devset.append(preprocess_example(example))
+
+ trainsetX = [dspy.Example(**x).with_inputs('title', 'abstract', 'context', 'labels') for x in trainset]
+ trainset = [dspy.Example(**x).with_inputs('title', 'abstract', 'context') for x in trainset]
+ devsetX = [dspy.Example(**x).with_inputs('title', 'abstract', 'context', 'labels') for x in devset]
+ devset = [dspy.Example(**x).with_inputs('title', 'abstract', 'context') for x in devset]
+
+ self.trainset = trainset
+ self.devset = devset
+
+ # Set up metrics
+ NUM_THREADS = 16
+
+ self.metric = metricR10
+
+ kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=15)
+ self.evaluarteR10 = Evaluate(devset=self.devset, metric=metricR10, **kwargs)
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return GroundedReactionExtractor(context_window=3000, max_windows=5, num_preds=1)
+
+ def get_metric(self):
+ return self.metric
\ No newline at end of file
diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.py
new file mode 100644
index 000000000..6724ebdcd
--- /dev/null
+++ b/testing/tasks/gsm8k.py
@@ -0,0 +1,77 @@
+import dspy
+from .base_task import BaseTask
+from dspy.datasets.gsm8k import gsm8k_metric
+import random
+import tqdm
+
+from datasets import load_dataset
+
+class CoT(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.prog = dspy.ChainOfThought("question -> answer")
+
+ def forward(self, question):
+ return self.prog(question=question)
+
+class GSM8KTask(BaseTask):
+ def __init__(self):
+ dataset = load_dataset("gsm8k", 'main')
+
+ hf_official_train = dataset['train']
+ hf_official_test = dataset['test']
+ official_train = []
+ official_test = []
+
+ for example in tqdm.tqdm(hf_official_train):
+ question = example['question']
+
+ answer = example['answer'].strip().split()
+ assert answer[-2] == '####'
+
+ gold_reasoning = ' '.join(answer[:-2])
+ answer = str(int(answer[-1].replace(',', '')))
+
+ official_train.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))
+
+ for example in tqdm.tqdm(hf_official_test):
+ question = example['question']
+
+ answer = example['answer'].strip().split()
+ assert answer[-2] == '####'
+
+ gold_reasoning = ' '.join(answer[:-2])
+ answer = str(int(answer[-1].replace(',', '')))
+
+ official_test.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))
+
+ rng = random.Random(0)
+ rng.shuffle(official_train)
+
+ rng = random.Random(0)
+ rng.shuffle(official_test)
+
+ trainset = official_train[:2000]
+ devset = official_train[2000:2500]
+ testset = official_test[:]
+
+ trainset = [dspy.Example(**x).with_inputs('question') for x in trainset]
+ devset = [dspy.Example(**x).with_inputs('question') for x in devset]
+ testset = [dspy.Example(**x).with_inputs('question') for x in testset]
+
+ # print(f"Trainset size: {len(trainset)}")
+ # print(f"Devset size: {len(devset)}")
+ # print(f"Testset size: {len(testset)}")
+
+ self.trainset= trainset
+ self.devset = devset
+ self.testset = testset
+ self.metric = gsm8k_metric
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return CoT()
+
+ def get_metric(self):
+ return self.metric
\ No newline at end of file
diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.py
new file mode 100644
index 000000000..19a5075fc
--- /dev/null
+++ b/testing/tasks/hotpotqa.py
@@ -0,0 +1,49 @@
+import dspy
+from dspy.evaluate import Evaluate
+from .base_task import BaseTask
+from dspy.datasets import HotPotQA
+
+class MultiHop(dspy.Module):
+ def __init__(self,passages_per_hop):
+ super().__init__()
+ self.retrieve = dspy.Retrieve(k = passages_per_hop)
+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")
+ self.generate_answer = dspy.ChainOfThought("context ,question->answer")
+
+ def forward (self,question) :
+ context = []
+ for hop in range(2):
+ query = self.generate_query(context = context, question = question).search_query
+ context += self.retrieve(query).passages
+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)
+
+class HotPotQATask(BaseTask):
+ def __init__(self):
+ # Load and configure the datasets.
+ hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)
+ self.trainset = [x.with_inputs('question') for x in hotpot_dataset.train]
+ self.devset = [x.with_inputs('question') for x in hotpot_dataset.dev]
+
+ # Set up metrics
+ NUM_THREADS = 16
+
+ metric_EM = dspy.evaluate.answer_exact_match
+ self.metric = metric_EM
+
+ def gold_passages_retrieved(example, pred, trace=None):
+ gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))
+ found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))
+ return gold_titles.issubset(found_titles)
+
+ kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=15)
+ self.evaluate_EM = Evaluate(devset=self.devset, metric=metric_EM, **kwargs)
+ self.evaluate_retrieval = Evaluate(devset=self.devset, metric=gold_passages_retrieved, **kwargs)
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return MultiHop(passages_per_hop=3) #TODO: make it so we can specify # of passages
+
+ def get_metric(self):
+ return self.metric
+
diff --git a/testing/tasks/scone.py b/testing/tasks/scone.py
new file mode 100644
index 000000000..e4cbad809
--- /dev/null
+++ b/testing/tasks/scone.py
@@ -0,0 +1,74 @@
+import glob
+import os
+import pandas as pd
+import dspy
+from .base_task import BaseTask
+import random
+
+def load_scone(dirname):
+ dfs = []
+ for filename in glob.glob(dirname + "/*.csv"):
+ df = pd.read_csv(filename, index_col=0)
+ df['category'] = os.path.basename(filename).replace(".csv", "")
+ dfs.append(df)
+ data_df = pd.concat(dfs)
+
+ def as_example(row):
+ # The 'one_scoped' file is from an earlier dataset, MoNLI, and
+ # so is formatted a bit differently:
+ suffix = '' if row['category'] == 'one_scoped' else '_edited'
+ # Reformat the hypothesis to be an embedded clause in a question:
+ hkey = 'sentence2' + suffix
+ question = row[hkey][0].lower() + row[hkey][1: ].strip(".")
+ question = f"Can we logically conclude for sure that {question}?"
+ # Binary task formulation:
+ label = "Yes" if row['gold_label' + suffix] == 'entailment' else "No"
+ return dspy.Example({
+ "context": row['sentence1' + suffix],
+ "question": question,
+ "answer": label,
+ "category": row['category']
+ }).with_inputs("context", "question")
+
+ return list(data_df.apply(as_example, axis=1).values)
+
+class ScoNeSignature(dspy.Signature):
+ ("""context, question -> answer""")
+
+ context = dspy.InputField()
+ question = dspy.InputField()
+ answer = dspy.OutputField(desc="Yes or No")
+
+class ScoNeCoT(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.generate_answer = dspy.ChainOfThought(ScoNeSignature)
+
+ def forward(self, context, question):
+ return self.generate_answer(context=context, question=question)
+
+class ScoNeTask(BaseTask):
+ def __init__(self):
+ # !git clone https://github.com/selenashe/ScoNe.git
+ if not os.path.exists("ScoNe"):
+ os.system("git clone https://github.com/selenashe/ScoNe.git")
+
+ # Load and configure the datasets.
+ all_train = load_scone("ScoNe/scone_nli/train")
+
+ random.seed(1)
+ random.shuffle(all_train)
+
+ # 1000 random train, 500 random dev:
+ self.trainset, self.devset = all_train[: 1000], all_train[1000: 1500]
+
+ metric_EM = dspy.evaluate.answer_exact_match
+ self.metric = metric_EM
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return ScoNeCoT()
+
+ def get_metric(self):
+ return self.metric
\ No newline at end of file
diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.py
new file mode 100644
index 000000000..ef3277ed0
--- /dev/null
+++ b/testing/tasks/tweet.py
@@ -0,0 +1,110 @@
+import dspy
+from dspy.datasets import HotPotQA
+from .base_task import BaseTask
+from dspy.evaluate import Evaluate
+from functools import lru_cache
+import openai
+from dotenv import load_dotenv
+import os
+
+class TweetSignature(dspy.Signature):
+ ("""Given context and a question, answer with a tweet""")
+
+ context = dspy.InputField()
+ question = dspy.InputField()
+ answer = dspy.OutputField(desc="Yes or No")
+
+class TweetCoT(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.generate_answer = dspy.ChainOfThought(TweetSignature)
+
+ def forward(self, context, question):
+ return self.generate_answer(context=context, question=question)
+
+class MultiHopTweet(dspy.Module):
+ def __init__(self,passages_per_hop):
+ super().__init__()
+ self.retrieve = dspy.Retrieve(k = passages_per_hop)
+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")
+ self.generate_answer = TweetCoT()
+
+ def forward (self,question) :
+ context = []
+ for hop in range(2):
+ query = self.generate_query(context = context, question = question).search_query
+ context += self.retrieve(query).passages
+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)
+
+# Define the signature for autoamtic assessments.
+class Assess(dspy.Signature):
+ """Assess the quality of a tweet along the specified dimension."""
+
+ context = dspy.InputField(desc='ignore if N/A')
+ assessed_text = dspy.InputField()
+ assessment_question = dspy.InputField()
+ assessment_answer = dspy.OutputField(desc="Yes or No")
+
+@lru_cache()
+def load_models():
+ load_dotenv() # This will load the .env file's variables
+
+ openai.api_key = os.environ.get('OPENAI_API_KEY')
+ openai.api_base = os.environ.get('OPENAI_API_BASE')
+ gpt4T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')
+ # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')
+ retrieve = dspy.Retrieve(k=5)
+ return gpt4T, retrieve
+
+METRIC = None
+def metric(gold, pred, trace=None):
+
+ gpt4T, retrieve = load_models()
+
+ question, answer, tweet = gold.question, gold.answer, pred.answer
+ context = retrieve(question).passages
+
+ engaging = "Does the assessed text make for a self-contained, engaging tweet?"
+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ correct = f"{correct} Does the assessed text above contain the gold answer?"
+
+ with dspy.context(lm=gpt4T):
+ faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)
+ correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)
+ engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)
+
+ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]
+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
+
+ if METRIC is not None:
+ if METRIC == 'correct': return correct
+ if METRIC == 'engaging': return engaging
+ if METRIC == 'faithful': return faithful
+
+ if trace is not None: return score >= 3
+ return score / 3.0
+
+class TweetTask(BaseTask):
+ def __init__(self):
+
+ # Load the dataset.
+ dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)
+
+ # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
+ self.trainset = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
+ self.devset = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
+
+ self.trainset = [x.with_inputs('question') for x in dataset.train]
+ self.devset = [x.with_inputs('question') for x in dataset.dev]
+
+ self.metric = metric
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return MultiHopTweet(passages_per_hop=3) #TODO: make it so we can specify # of passages
+
+ def get_metric(self):
+ return self.metric
+
\ No newline at end of file
diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.py
new file mode 100644
index 000000000..2e237d0c2
--- /dev/null
+++ b/testing/tasks/tweet_metric.py
@@ -0,0 +1,148 @@
+import dspy
+from dspy.datasets import HotPotQA
+from .base_task import BaseTask
+from dspy.evaluate import Evaluate
+from dspy import Example
+from functools import lru_cache
+import openai
+from dotenv import load_dotenv
+import os
+import uuid
+from tqdm import tqdm
+
+class TweetSignature(dspy.Signature):
+ ("""Given context and a question, answer with a tweet""")
+
+ context = dspy.InputField()
+ question = dspy.InputField()
+ answer = dspy.OutputField(desc="Yes or No")
+
+class TweetCoT(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.generate_answer = dspy.ChainOfThought(TweetSignature)
+
+ def forward(self, context, question):
+ return self.generate_answer(context=context, question=question)
+
+class MultiHopTweet(dspy.Module):
+ def __init__(self,passages_per_hop):
+ super().__init__()
+ self.retrieve = dspy.Retrieve(k = passages_per_hop)
+ self.generate_query = dspy.ChainOfThought("context ,question->search_query")
+ self.generate_answer = TweetCoT()
+
+ def forward (self,question) :
+ context = []
+ for hop in range(2):
+ query = self.generate_query(context = context, question = question).search_query
+ context += self.retrieve(query).passages
+ return dspy.Prediction(context=context, answer=self.generate_answer(context = context , question = question).answer)
+
+# Define the signature for autoamtic assessments.
+class Assess(dspy.Signature):
+ """Assess the quality of a tweet along the specified dimension."""
+
+ context = dspy.InputField(desc='ignore if N/A')
+ assessed_text = dspy.InputField()
+ assessment_question = dspy.InputField()
+ assessment_answer = dspy.OutputField(desc="Yes or No")
+
+@lru_cache()
+def load_models():
+ load_dotenv() # This will load the .env file's variables
+
+ openai.api_key = os.environ.get('OPENAI_API_KEY')
+ openai.api_base = os.environ.get('OPENAI_API_BASE')
+ gpt3T = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=1000, model_type='chat')
+ gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=1000, model_type='chat')
+ retrieve = dspy.Retrieve(k=5)
+ return gpt3T, gpt4T, retrieve
+
+METRIC = None
+def metric(gold, pred, trace=None):
+ gpt3T, gpt4T, retrieve = load_models()
+
+ question, answer, tweet, context = gold.question, gold.answer, gold.tweet, gold.context
+ score_pred = pred.score
+
+ engaging = "Does the assessed text make for a self-contained, engaging tweet?"
+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ correct = f"{correct} Does the assessed text above contain the gold answer?"
+
+ with dspy.context(lm=gpt3T): # TODO Update to GPT4
+ faithful = dspy.Predict(Assess)(context=context, assessed_text=tweet, assessment_question=faithful)
+ correct = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=correct)
+ engaging = dspy.Predict(Assess)(context='N/A', assessed_text=tweet, assessment_question=engaging)
+
+ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]
+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
+
+ return 1 - abs(score - score_pred) # We want a score we can maximize, so take the negative L1 norm and add 1
+
+class TweetMetric(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.engaging = dspy.Predict(Assess)
+ self.faithful = dspy.Predict(Assess)
+ self.correct = dspy.Predict(Assess)
+
+ def forward (self, tweet, context, question, answer) :
+ engaging = "Does the assessed text make for a self-contained, engaging tweet?"
+ faithful = "Is the assessed text grounded in the context? Say no if it includes significant facts not in the context."
+ correct = f"The text above is should answer `{question}`. The gold answer is `{answer}`."
+ correct = f"{correct} Does the assessed text above contain the gold answer?"
+
+ faithful = self.faithful(context=context, assessed_text=tweet, assessment_question=faithful)
+ correct = self.correct(context='N/A', assessed_text=tweet, assessment_question=correct)
+ engaging = self.engaging(context='N/A', assessed_text=tweet, assessment_question=engaging)
+
+ correct, engaging, faithful = [m.assessment_answer.split()[0].lower() == 'yes' for m in [correct, engaging, faithful]]
+ score = (correct + engaging + faithful) if correct and (len(tweet) <= 280) else 0
+
+ return dspy.Prediction(score= score/3.0)
+
+class TweetMetricTask(BaseTask):
+ def __init__(self):
+
+ # Load the dataset.
+ dataset = HotPotQA(train_seed=1, train_size=500, eval_seed=2023, dev_size=200, test_size=0, keep_details=True)
+
+ # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
+ trainset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.train]
+ devset_temp = [x.without('id', 'type').with_inputs('question') for x in dataset.dev]
+ self.trainset = []
+ self.devset = []
+
+ gpt3T, gpt4T, retrieve = load_models()
+
+ with dspy.context(lm=gpt3T):
+ for ex in tqdm(trainset_temp, desc="Preprocessing Trainset"):
+ context = retrieve(ex.question).passages
+ question = ex.question
+ answer = ex.answer
+ tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer
+ example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}
+
+ self.trainset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='train').with_inputs('context', 'question', 'answer', 'tweet'))
+
+ for ex in tqdm(devset_temp, desc="Preprocessing Devset"):
+ context = retrieve(ex.question).passages
+ question = ex.question
+ answer = ex.answer
+ tweet = MultiHopTweet(passages_per_hop=3)(ex.question).answer
+ example = {'context': context, 'question': question, 'answer': answer, 'tweet': tweet}
+
+ self.devset.append(Example(**example, dspy_uuid=str(uuid.uuid4()), dspy_split='dev').with_inputs('context', 'question', 'answer', 'tweet'))
+
+ self.metric = metric
+
+ self.set_splits(TRAIN_NUM=100, DEV_NUM=100, EVAL_NUM=100)
+
+ def get_program(self):
+ return TweetMetric()
+
+ def get_metric(self):
+ return self.metric
+
\ No newline at end of file