DSPy: The framework for programming—not prompting—language models
Python
Multi-Agent Example
Omar Khattab784b75e43585
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 03c9623ed..a16c35653 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -152,6 +152,7 @@ class Evaluate:del thread_stacks[threading.get_ident()]devset = list(enumerate(devset))+ tqdm.tqdm._instances.clear()if num_threads == 1:reordered_devset, ncorrect, ntotal = self._execute_single_thread(wrapped_program, devset, display_progress)@@ -254,7 +255,9 @@ def configure_dataframe_display(df, metric_name):pd.set_option("display.width", 400) # Adjust# df[metric_name] = df[metric_name].apply(lambda x: f'✔️ [{x}]' if x is True else f'❌ [{x}]')- df.loc[:, metric_name] = df[metric_name].apply(lambda x: f"✔️ [{x}]" if x is True else f"{x}")+ # df.loc[:, metric_name] = df[metric_name].apply(lambda x: f"✔️ [{x}]" if x is True else f"{x}")+ df[metric_name] = df[metric_name].apply(lambda x: f"✔️ [{x}]" if x else str(x))+# Return styled DataFramereturn df.style.set_table_styles(diff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex cedd49c77..5077d7219 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -99,7 +99,7 @@ class ReAct(Module):output[f"Observation_{hop+1}"] = ("Failed to parse action. Bad formatting or incorrect action name.")- raise e+ # raise edef forward(self, **kwargs):args = {key: kwargs[key] for key in self.input_fields.keys() if key in kwargs}@@ -112,5 +112,7 @@ class ReAct(Module):breakargs.update(output)+ observations = [args[key] for key in args if key.startswith("Observation")]+# assumes only 1 output field for now - TODO: handling for multiple output fields- return dspy.Prediction(**{list(self.output_fields.keys())[0]: action_val or ""})+ return dspy.Prediction(observations=observations, **{list(self.output_fields.keys())[0]: action_val or ""})diff --git a/examples/agents/multi_agent.ipynb b/examples/agents/multi_agent.ipynbnew file mode 100644index 000000000..7678cfbd1--- /dev/null+++ b/examples/agents/multi_agent.ipynb@@ -0,0 +1,861 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "<img src=\"../../docs/images/DSPy8.png\" alt=\"DSPy7 Image\" height=\"120\"/>\n",+ "\n",+ "### Multi-Agent DSPy Programs: Bootstrapping & Aggregating Multiple `ReAct` Agents\n",+ "\n",+ "This is a quick (somewhat advanced) example of DSPy. You're given a hard QA task and an agent architecture (`dspy.ReAct`), how do you get high scores without tinkering with prompts?\n",+ "\n",+ "There are many ways, but this notebook shows one complex strategy that DSPy makes near-trivial to achieve: we'll automatically bootstrap five different highly-effective prompts for ReAct, then optimize an aggregator that combines their powers.\n",+ "\n",+ "As is usually the case with DSPy, the code to do this is probably shorter than describing it in English, so let's jump right into that."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 0) TLDR.\n",+ "\n",+ "We'll build a ReAct agent in DSPy that scores 30% accuracy on a retrieval-based question answering task.\n",+ "\n",+ "Then, we'll optimize it with `BootstrapFewShotWithRandomSearch` to get 46% accuracy.\n",+ "\n",+ "Then, we'll build a multi-agent aggregator over five different optimized versions of the agent.\n",+ "\n",+ "Our unoptimized aggregator will score 26%. It doesn't understand the task. Hence, we'll optimize the aggregator too.\n",+ "\n",+ "We'll end up with an optimized multi-agent system that scores a whopping 60% accuracy on the same task.\n",+ "\n",+ "The core portion of the code to do this can be fit into 10 lines of DSPy, but we'll sprinkle some short explanations below."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 1) Setting Up.\n",+ "\n",+ "We'll configure the language model (GPT-3.5) and the retrieval model (ColBERTv2 over Wikipedia)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "import dspy\n",+ "from dspy.evaluate import Evaluate\n",+ "from dspy.datasets.hotpotqa import HotPotQA\n",+ "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",+ "\n",+ "gpt3 = dspy.OpenAI('gpt-3.5-turbo-0125', max_tokens=1000)\n",+ "colbert = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",+ "dspy.configure(lm=gpt3, rm=colbert)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 2) Loading some data.\n",+ "\n",+ "We'll load 150 examples for training (`trainset`), 50 examples for validation & optimization (`valset`), and 300 examples for evaluation (`devset`)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "Example({'question': 'At My Window was released by which American singer-songwriter?', 'answer': 'John Townes Van Zandt'}) (input_keys={'question'})"+ ]+ },+ "execution_count": 2,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "dataset = HotPotQA(train_seed=1, train_size=200, eval_seed=2023, dev_size=300, test_size=0)\n",+ "trainset = [x.with_inputs('question') for x in dataset.train[0:150]]\n",+ "valset = [x.with_inputs('question') for x in dataset.train[150:200]]\n",+ "devset = [x.with_inputs('question') for x in dataset.dev]\n",+ "\n",+ "# show an example datapoint; it's just a question-answer pair\n",+ "trainset[0]"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 3) ReAct Agent.\n",+ "\n",+ "Our agent will just be a DSPy ReAct agent that takes a `question` and outputs the `answer` by using a ColBERTv2 retrieval tool."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "agent = dspy.ReAct(\"question -> answer\", tools=[dspy.Retrieve(k=1)])"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's evaluate this **unoptimized** ReAct agent on the `devset`."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 91 / 300 (30.3): 100%|██████████| 300/300 [00:01<00:00, 161.84it/s]\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_a7036 th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_a7036 td {\n",+ " text-align: left;\n",+ "}\n",+ "#T_a7036_row0_col0, #T_a7036_row0_col1, #T_a7036_row0_col2, #T_a7036_row0_col3, #T_a7036_row0_col4, #T_a7036_row0_col5, #T_a7036_row1_col0, #T_a7036_row1_col1, #T_a7036_row1_col2, #T_a7036_row1_col3, #T_a7036_row1_col4, #T_a7036_row1_col5, #T_a7036_row2_col0, #T_a7036_row2_col1, #T_a7036_row2_col2, #T_a7036_row2_col3, #T_a7036_row2_col4, #T_a7036_row2_col5, #T_a7036_row3_col0, #T_a7036_row3_col1, #T_a7036_row3_col2, #T_a7036_row3_col3, #T_a7036_row3_col4, #T_a7036_row3_col5, #T_a7036_row4_col0, #T_a7036_row4_col1, #T_a7036_row4_col2, #T_a7036_row4_col3, #T_a7036_row4_col4, #T_a7036_row4_col5 {\n",+ " text-align: left;\n",+ " white-space: pre-wrap;\n",+ " word-wrap: break-word;\n",+ " max-width: 400px;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_a7036\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th id=\"T_a7036_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",+ " <th id=\"T_a7036_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",+ " <th id=\"T_a7036_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",+ " <th id=\"T_a7036_level0_col3\" class=\"col_heading level0 col3\" >observations</th>\n",+ " <th id=\"T_a7036_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",+ " <th id=\"T_a7036_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_a7036_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_a7036_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",+ " <td id=\"T_a7036_row0_col1\" class=\"data row0 col1\" >no</td>\n",+ " <td id=\"T_a7036_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",+ " <td id=\"T_a7036_row0_col3\" class=\"data row0 col3\" >[['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",+ " <td id=\"T_a7036_row0_col4\" class=\"data row0 col4\" >No, Cangzhou is in the Hebei province, while Qionghai is in the Hainan province of China.</td>\n",+ " <td id=\"T_a7036_row0_col5\" class=\"data row0 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_a7036_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_a7036_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",+ " <td id=\"T_a7036_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",+ " <td id=\"T_a7036_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",+ " <td id=\"T_a7036_row1_col3\" class=\"data row1 col3\" >[[\"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to...</td>\n",+ " <td id=\"T_a7036_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",+ " <td id=\"T_a7036_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_a7036_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_a7036_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",+ " <td id=\"T_a7036_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",+ " <td id=\"T_a7036_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",+ " <td id=\"T_a7036_row2_col3\" class=\"data row2 col3\" >[['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",+ " <td id=\"T_a7036_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",+ " <td id=\"T_a7036_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_a7036_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_a7036_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",+ " <td id=\"T_a7036_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",+ " <td id=\"T_a7036_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",+ " <td id=\"T_a7036_row3_col3\" class=\"data row3 col3\" >[[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",+ " <td id=\"T_a7036_row3_col4\" class=\"data row3 col4\" >Tweed River</td>\n",+ " <td id=\"T_a7036_row3_col5\" class=\"data row3 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_a7036_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_a7036_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",+ " <td id=\"T_a7036_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",+ " <td id=\"T_a7036_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",+ " <td id=\"T_a7036_row4_col3\" class=\"data row4 col3\" >[['Æthelweard (son of Alfred) | Æthelweard (d. 920 or 922) was the younger son of King Alfred the Great and Ealhswith.']]</td>\n",+ " <td id=\"T_a7036_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",+ " <td id=\"T_a7036_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f218fff3730>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/html": [+ "\n",+ " <div style='\n",+ " text-align: center;\n",+ " font-size: 16px;\n",+ " font-weight: bold;\n",+ " color: #555;\n",+ " margin: 10px 0;'>\n",+ " ... 295 more rows not displayed ...\n",+ " </div>\n",+ " "+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/plain": [+ "30.33"+ ]+ },+ "execution_count": 4,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "# Set up an evaluator on the first 300 examples of the devset.\n",+ "config = dict(num_threads=8, display_progress=True, display_table=5)\n",+ "evaluate = Evaluate(devset=devset, metric=dspy.evaluate.answer_exact_match, **config)\n",+ "\n",+ "evaluate(agent)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 4) Optimized ReAct.\n",+ "\n",+ "Let's use DSPy's simple `BootstrapFewShotWithRandomSearch` optimizer to create successful examples of the ReAct program and attempt to optimize the prompts using those constructed examples. In the future, we could try more sophisticated DSPy optimizers too, like `MIPRO`.\n",+ "\n",+ "We'll bootstrap 20 programs that way. Examples will be bootstrapped starting from the `trainset` and optimized over our tiny `valset`. We'll evaluate later on the `devset`."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 14 / 50 (28.0): 100%|██████████| 50/50 [00:00<00:00, 151.32it/s]\n",+ "Average Metric: 14 / 50 (28.0): 100%|██████████| 50/50 [00:00<00:00, 1191.35it/s]\n",+ " 4%|▍ | 6/150 [00:00<00:00, 216.36it/s]\n",+ "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 158.43it/s]\n",+ " 3%|▎ | 4/150 [00:00<00:00, 258.73it/s]\n",+ "Average Metric: 21 / 50 (42.0): 100%|██████████| 50/50 [00:00<00:00, 184.63it/s]\n",+ " 3%|▎ | 4/150 [00:00<00:01, 125.61it/s]\n",+ "Average Metric: 24 / 50 (48.0): 100%|██████████| 50/50 [00:00<00:00, 130.39it/s]\n",+ " 1%|▏ | 2/150 [00:00<00:00, 213.13it/s]\n",+ "Average Metric: 20 / 50 (40.0): 100%|██████████| 50/50 [00:00<00:00, 158.40it/s]\n",+ " 3%|▎ | 4/150 [00:00<00:00, 387.38it/s]\n",+ "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:00<00:00, 168.50it/s]\n",+ " 4%|▍ | 6/150 [00:00<00:00, 201.99it/s]\n",+ "Average Metric: 12 / 50 (24.0): 100%|██████████| 50/50 [00:00<00:00, 152.09it/s]\n",+ " 6%|▌ | 9/150 [00:00<00:00, 203.21it/s]\n",+ "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 231.82it/s]\n",+ " 8%|▊ | 12/150 [00:00<00:00, 275.48it/s]\n",+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 165.50it/s]\n",+ " 9%|▊ | 13/150 [00:00<00:00, 238.67it/s]\n",+ "Average Metric: 20 / 50 (40.0): 100%|██████████| 50/50 [00:00<00:00, 167.18it/s]\n",+ " 1%| | 1/150 [00:00<00:00, 624.06it/s]\n",+ "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:00<00:00, 963.47it/s] \n",+ " 3%|▎ | 5/150 [00:00<00:00, 335.88it/s]\n",+ "Average Metric: 21 / 50 (42.0): 100%|██████████| 50/50 [00:00<00:00, 173.26it/s]\n",+ " 1%| | 1/150 [00:00<00:00, 196.26it/s]\n",+ "Average Metric: 25 / 50 (50.0): 100%|██████████| 50/50 [00:00<00:00, 156.35it/s]\n",+ " 7%|▋ | 10/150 [00:00<00:00, 415.36it/s]\n",+ "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 180.18it/s]\n",+ " 11%|█▏ | 17/150 [00:00<00:00, 265.33it/s]\n",+ "Average Metric: 23 / 50 (46.0): 100%|██████████| 50/50 [00:00<00:00, 94.46it/s] \n",+ " 4%|▍ | 6/150 [00:00<00:00, 273.37it/s]\n",+ "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 193.94it/s]\n",+ " 1%|▏ | 2/150 [00:00<00:00, 290.68it/s]\n",+ "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:00<00:00, 1055.39it/s]\n",+ " 1%| | 1/150 [00:00<00:00, 268.11it/s]\n",+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 148.74it/s]\n",+ " 3%|▎ | 4/150 [00:00<00:00, 204.06it/s]\n",+ "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:00<00:00, 183.84it/s]\n",+ " 9%|▉ | 14/150 [00:00<00:00, 343.80it/s]\n",+ "Average Metric: 23 / 50 (46.0): 100%|██████████| 50/50 [00:00<00:00, 148.82it/s]\n",+ " 1%|▏ | 2/150 [00:00<00:00, 590.25it/s]\n",+ "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 223.29it/s]\n",+ " 1%| | 1/150 [00:00<00:00, 169.02it/s]\n",+ "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 128.94it/s]\n"+ ]+ }+ ],+ "source": [+ "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",+ "\n",+ "config = dict(max_bootstrapped_demos=2, max_labeled_demos=0, num_candidate_programs=20, num_threads=8)\n",+ "tp = BootstrapFewShotWithRandomSearch(metric=dspy.evaluate.answer_exact_match, **config)\n",+ "optimized_react = tp.compile(agent, trainset=trainset, valset=valset)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 138 / 300 (46.0): 100%|██████████| 300/300 [00:02<00:00, 137.82it/s]\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_fe347 th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_fe347 td {\n",+ " text-align: left;\n",+ "}\n",+ "#T_fe347_row0_col0, #T_fe347_row0_col1, #T_fe347_row0_col2, #T_fe347_row0_col3, #T_fe347_row0_col4, #T_fe347_row0_col5, #T_fe347_row1_col0, #T_fe347_row1_col1, #T_fe347_row1_col2, #T_fe347_row1_col3, #T_fe347_row1_col4, #T_fe347_row1_col5, #T_fe347_row2_col0, #T_fe347_row2_col1, #T_fe347_row2_col2, #T_fe347_row2_col3, #T_fe347_row2_col4, #T_fe347_row2_col5, #T_fe347_row3_col0, #T_fe347_row3_col1, #T_fe347_row3_col2, #T_fe347_row3_col3, #T_fe347_row3_col4, #T_fe347_row3_col5, #T_fe347_row4_col0, #T_fe347_row4_col1, #T_fe347_row4_col2, #T_fe347_row4_col3, #T_fe347_row4_col4, #T_fe347_row4_col5 {\n",+ " text-align: left;\n",+ " white-space: pre-wrap;\n",+ " word-wrap: break-word;\n",+ " max-width: 400px;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_fe347\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th id=\"T_fe347_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",+ " <th id=\"T_fe347_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",+ " <th id=\"T_fe347_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",+ " <th id=\"T_fe347_level0_col3\" class=\"col_heading level0 col3\" >observations</th>\n",+ " <th id=\"T_fe347_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",+ " <th id=\"T_fe347_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_fe347_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_fe347_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",+ " <td id=\"T_fe347_row0_col1\" class=\"data row0 col1\" >no</td>\n",+ " <td id=\"T_fe347_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",+ " <td id=\"T_fe347_row0_col3\" class=\"data row0 col3\" >[['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",+ " <td id=\"T_fe347_row0_col4\" class=\"data row0 col4\" >no</td>\n",+ " <td id=\"T_fe347_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_fe347_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_fe347_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",+ " <td id=\"T_fe347_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",+ " <td id=\"T_fe347_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",+ " <td id=\"T_fe347_row1_col3\" class=\"data row1 col3\" >[['2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was...</td>\n",+ " <td id=\"T_fe347_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",+ " <td id=\"T_fe347_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_fe347_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_fe347_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",+ " <td id=\"T_fe347_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",+ " <td id=\"T_fe347_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",+ " <td id=\"T_fe347_row2_col3\" class=\"data row2 col3\" >[['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",+ " <td id=\"T_fe347_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",+ " <td id=\"T_fe347_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_fe347_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_fe347_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",+ " <td id=\"T_fe347_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",+ " <td id=\"T_fe347_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",+ " <td id=\"T_fe347_row3_col3\" class=\"data row3 col3\" >[[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",+ " <td id=\"T_fe347_row3_col4\" class=\"data row3 col4\" >Crichton Collegiate Church is located in Midlothian, Scotland, near the hamlet of Crichton, about 7.5 miles south of Edinburgh.</td>\n",+ " <td id=\"T_fe347_row3_col5\" class=\"data row3 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_fe347_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_fe347_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",+ " <td id=\"T_fe347_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",+ " <td id=\"T_fe347_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",+ " <td id=\"T_fe347_row4_col3\" class=\"data row4 col3\" >[['Æthelweard (son of Alfred) | Æthelweard (d. 920 or 922) was the younger son of King Alfred the Great and Ealhswith.'], ['Æthelstan of Kent |...</td>\n",+ " <td id=\"T_fe347_row4_col4\" class=\"data row4 col4\" >Alfred the Great</td>\n",+ " <td id=\"T_fe347_row4_col5\" class=\"data row4 col5\" >False</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f216eaed6f0>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/html": [+ "\n",+ " <div style='\n",+ " text-align: center;\n",+ " font-size: 16px;\n",+ " font-weight: bold;\n",+ " color: #555;\n",+ " margin: 10px 0;'>\n",+ " ... 295 more rows not displayed ...\n",+ " </div>\n",+ " "+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/plain": [+ "46.0"+ ]+ },+ "execution_count": 6,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "evaluate(optimized_react)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 5) Zero-Shot Aggregator.\n",+ "\n",+ "Let's now extract the best five bootstrapped ReAct programs. We'll build a simple DSPy aggregator that runs all of them then produces a final answer."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dsp.utils import flatten, deduplicate\n",+ "\n",+ "# the best-performing five ReAct programs from the optimization process\n",+ "AGENTS = [x[-1] for x in optimized_react.candidate_programs[:5]]\n",+ "\n",+ "class Aggregator(dspy.Module):\n",+ "\tdef __init__(self, temperature=0.0):\n",+ "\t\tself.aggregate = dspy.ChainOfThought('context, question -> answer')\n",+ "\t\tself.temperature = temperature\n",+ "\n",+ "\tdef forward(self, question):\n",+ "\t\t# Run all five agents with high temperature, then extract and deduplicate their observed contexts\n",+ "\t\twith dspy.context(lm=gpt3.copy(temperature=self.temperature)):\n",+ "\t\t\tpreds = [agent(question=question) for agent in AGENTS]\n",+ "\t\t\tcontext = deduplicate(flatten([flatten(p.observations) for p in preds]))\n",+ "\n",+ "\t\t# Run the aggregation step to produce a final answer\n",+ "\t\treturn self.aggregate(context=context, question=question)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's quickly evaluate the aggregator prior to optimization."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 78 / 300 (26.0): 100%|██████████| 300/300 [00:06<00:00, 45.38it/s]\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_dec17 th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_dec17 td {\n",+ " text-align: left;\n",+ "}\n",+ "#T_dec17_row0_col0, #T_dec17_row0_col1, #T_dec17_row0_col2, #T_dec17_row0_col3, #T_dec17_row0_col4, #T_dec17_row0_col5, #T_dec17_row1_col0, #T_dec17_row1_col1, #T_dec17_row1_col2, #T_dec17_row1_col3, #T_dec17_row1_col4, #T_dec17_row1_col5, #T_dec17_row2_col0, #T_dec17_row2_col1, #T_dec17_row2_col2, #T_dec17_row2_col3, #T_dec17_row2_col4, #T_dec17_row2_col5, #T_dec17_row3_col0, #T_dec17_row3_col1, #T_dec17_row3_col2, #T_dec17_row3_col3, #T_dec17_row3_col4, #T_dec17_row3_col5, #T_dec17_row4_col0, #T_dec17_row4_col1, #T_dec17_row4_col2, #T_dec17_row4_col3, #T_dec17_row4_col4, #T_dec17_row4_col5 {\n",+ " text-align: left;\n",+ " white-space: pre-wrap;\n",+ " word-wrap: break-word;\n",+ " max-width: 400px;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_dec17\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th id=\"T_dec17_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",+ " <th id=\"T_dec17_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",+ " <th id=\"T_dec17_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",+ " <th id=\"T_dec17_level0_col3\" class=\"col_heading level0 col3\" >rationale</th>\n",+ " <th id=\"T_dec17_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",+ " <th id=\"T_dec17_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_dec17_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_dec17_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",+ " <td id=\"T_dec17_row0_col1\" class=\"data row0 col1\" >no</td>\n",+ " <td id=\"T_dec17_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",+ " <td id=\"T_dec17_row0_col3\" class=\"data row0 col3\" >determine if both Cangzhou and Qionghai are in the Hebei province of China. We need to carefully analyze the information provided in the context to...</td>\n",+ " <td id=\"T_dec17_row0_col4\" class=\"data row0 col4\" >No, only Cangzhou is in the Hebei province of China. Qionghai is located in Hainan province.</td>\n",+ " <td id=\"T_dec17_row0_col5\" class=\"data row0 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_dec17_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_dec17_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",+ " <td id=\"T_dec17_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",+ " <td id=\"T_dec17_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",+ " <td id=\"T_dec17_row1_col3\" class=\"data row1 col3\" >produce the answer. We know that Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. Looking at the context provided, we...</td>\n",+ " <td id=\"T_dec17_row1_col4\" class=\"data row1 col4\" >The 2017 NHL Expansion Draft conducted by the National Hockey League filled the roster of the Vegas Golden Knights, including selecting Marc-Andre Fleury for the...</td>\n",+ " <td id=\"T_dec17_row1_col5\" class=\"data row1 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_dec17_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_dec17_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",+ " <td id=\"T_dec17_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",+ " <td id=\"T_dec17_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",+ " <td id=\"T_dec17_row2_col3\" class=\"data row2 col3\" >identify the retired Canadian professional ice hockey player and current general manager of the Tampa Bay Lightning of the National Hockey League (NHL) whose retirement...</td>\n",+ " <td id=\"T_dec17_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",+ " <td id=\"T_dec17_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_dec17_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_dec17_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",+ " <td id=\"T_dec17_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",+ " <td id=\"T_dec17_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",+ " <td id=\"T_dec17_row3_col3\" class=\"data row3 col3\" >identify the river near the Crichton Collegiate Church. We know that the church is situated in Midlothian, Scotland, and the River Esk flows through Midlothian...</td>\n",+ " <td id=\"T_dec17_row3_col4\" class=\"data row3 col4\" >The River Esk</td>\n",+ " <td id=\"T_dec17_row3_col5\" class=\"data row3 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_dec17_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_dec17_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",+ " <td id=\"T_dec17_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",+ " <td id=\"T_dec17_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",+ " <td id=\"T_dec17_row4_col3\" class=\"data row4 col3\" >produce the answer. We know from the context that Ealhswith had a son named Æthelweard in the 10th century A.D. Now, looking at the information...</td>\n",+ " <td id=\"T_dec17_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",+ " <td id=\"T_dec17_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f218ff8e8c0>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/html": [+ "\n",+ " <div style='\n",+ " text-align: center;\n",+ " font-size: 16px;\n",+ " font-weight: bold;\n",+ " color: #555;\n",+ " margin: 10px 0;'>\n",+ " ... 295 more rows not displayed ...\n",+ " </div>\n",+ " "+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/plain": [+ "26.0"+ ]+ },+ "execution_count": 8,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "aggregator = Aggregator()\n",+ "evaluate(aggregator)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 6) Optimized Aggregator."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 153.98it/s]\n",+ "Average Metric: 27 / 50 (54.0): 100%|██████████| 50/50 [00:00<00:00, 82.75it/s]\n",+ " 3%|▎ | 4/150 [00:00<00:03, 45.32it/s]\n",+ "Average Metric: 28 / 50 (56.0): 100%|██████████| 50/50 [00:00<00:00, 156.28it/s]\n",+ " 1%|▏ | 2/150 [00:00<00:03, 39.99it/s]\n",+ "Average Metric: 28 / 50 (56.0): 100%|██████████| 50/50 [00:00<00:00, 162.26it/s]\n",+ " 1%| | 1/150 [00:00<00:02, 51.23it/s]\n",+ "Average Metric: 26 / 50 (52.0): 100%|██████████| 50/50 [00:00<00:00, 158.64it/s]\n",+ " 1%| | 1/150 [00:00<00:00, 155.47it/s]\n",+ "Average Metric: 28 / 50 (56.0): 100%|██████████| 50/50 [00:00<00:00, 159.96it/s]\n",+ " 1%| | 1/150 [00:00<00:04, 31.56it/s]\n",+ "Average Metric: 27 / 50 (54.0): 100%|██████████| 50/50 [00:00<00:00, 143.11it/s]\n",+ " 1%| | 1/150 [00:00<00:03, 43.19it/s]\n",+ "Average Metric: 29 / 50 (58.0): 100%|██████████| 50/50 [00:00<00:00, 163.95it/s]\n",+ " 1%|▏ | 2/150 [00:00<00:04, 31.94it/s]\n",+ "Average Metric: 25 / 50 (50.0): 100%|██████████| 50/50 [00:00<00:00, 197.13it/s]\n",+ " 2%|▏ | 3/150 [00:00<00:04, 35.74it/s]\n",+ "Average Metric: 27 / 50 (54.0): 100%|██████████| 50/50 [00:00<00:00, 161.65it/s]\n",+ " 5%|▍ | 7/150 [00:00<00:03, 40.74it/s]\n",+ "Average Metric: 29 / 50 (58.0): 100%|██████████| 50/50 [00:00<00:00, 176.77it/s]\n",+ " 1%| | 1/150 [00:00<00:02, 65.18it/s]\n",+ "Average Metric: 28 / 50 (56.0): 100%|██████████| 50/50 [00:00<00:00, 153.30it/s]\n",+ " 2%|▏ | 3/150 [00:00<00:02, 59.95it/s]\n",+ "Average Metric: 30 / 50 (60.0): 100%|██████████| 50/50 [00:00<00:00, 148.52it/s]\n"+ ]+ }+ ],+ "source": [+ "kwargs = dict(max_bootstrapped_demos=2, max_labeled_demos=6, num_candidate_programs=10, num_threads=8)\n",+ "tp = BootstrapFewShotWithRandomSearch(metric=dspy.evaluate.answer_exact_match, **kwargs)\n",+ "optimized_aggregator = tp.compile(aggregator, trainset=trainset, valset=valset)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 180 / 300 (60.0): 100%|██████████| 300/300 [00:07<00:00, 42.10it/s]\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_50ac6 th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_50ac6 td {\n",+ " text-align: left;\n",+ "}\n",+ "#T_50ac6_row0_col0, #T_50ac6_row0_col1, #T_50ac6_row0_col2, #T_50ac6_row0_col3, #T_50ac6_row0_col4, #T_50ac6_row0_col5, #T_50ac6_row1_col0, #T_50ac6_row1_col1, #T_50ac6_row1_col2, #T_50ac6_row1_col3, #T_50ac6_row1_col4, #T_50ac6_row1_col5, #T_50ac6_row2_col0, #T_50ac6_row2_col1, #T_50ac6_row2_col2, #T_50ac6_row2_col3, #T_50ac6_row2_col4, #T_50ac6_row2_col5, #T_50ac6_row3_col0, #T_50ac6_row3_col1, #T_50ac6_row3_col2, #T_50ac6_row3_col3, #T_50ac6_row3_col4, #T_50ac6_row3_col5, #T_50ac6_row4_col0, #T_50ac6_row4_col1, #T_50ac6_row4_col2, #T_50ac6_row4_col3, #T_50ac6_row4_col4, #T_50ac6_row4_col5 {\n",+ " text-align: left;\n",+ " white-space: pre-wrap;\n",+ " word-wrap: break-word;\n",+ " max-width: 400px;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_50ac6\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th id=\"T_50ac6_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",+ " <th id=\"T_50ac6_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",+ " <th id=\"T_50ac6_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",+ " <th id=\"T_50ac6_level0_col3\" class=\"col_heading level0 col3\" >rationale</th>\n",+ " <th id=\"T_50ac6_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",+ " <th id=\"T_50ac6_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_50ac6_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_50ac6_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",+ " <td id=\"T_50ac6_row0_col1\" class=\"data row0 col1\" >no</td>\n",+ " <td id=\"T_50ac6_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",+ " <td id=\"T_50ac6_row0_col3\" class=\"data row0 col3\" >produce the answer. From the context, we know that Cangzhou is a prefecture-level city in eastern Hebei province, while Qionghai is one of the seven...</td>\n",+ " <td id=\"T_50ac6_row0_col4\" class=\"data row0 col4\" >no</td>\n",+ " <td id=\"T_50ac6_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_50ac6_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_50ac6_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",+ " <td id=\"T_50ac6_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",+ " <td id=\"T_50ac6_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",+ " <td id=\"T_50ac6_row1_col3\" class=\"data row1 col3\" >produce the answer. From the context, we know that Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. The draft that...</td>\n",+ " <td id=\"T_50ac6_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",+ " <td id=\"T_50ac6_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_50ac6_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_50ac6_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",+ " <td id=\"T_50ac6_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",+ " <td id=\"T_50ac6_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",+ " <td id=\"T_50ac6_row2_col3\" class=\"data row2 col3\" >produce the answer. We know from the context that Steve Yzerman is a Canadian retired professional ice hockey player and the current general manager of...</td>\n",+ " <td id=\"T_50ac6_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",+ " <td id=\"T_50ac6_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_50ac6_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_50ac6_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",+ " <td id=\"T_50ac6_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",+ " <td id=\"T_50ac6_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",+ " <td id=\"T_50ac6_row3_col3\" class=\"data row3 col3\" >produce the answer. We know that Crichton Collegiate Church is located in Midlothian, Scotland, near the hamlet of Crichton. Since it is close to Edinburgh,...</td>\n",+ " <td id=\"T_50ac6_row3_col4\" class=\"data row3 col4\" >River Esk</td>\n",+ " <td id=\"T_50ac6_row3_col5\" class=\"data row3 col5\" >False</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_50ac6_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_50ac6_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",+ " <td id=\"T_50ac6_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",+ " <td id=\"T_50ac6_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",+ " <td id=\"T_50ac6_row4_col3\" class=\"data row4 col3\" >produce the answer. From the context, we know that Ealhswith was the wife of King Alfred the Great. Therefore, in the 10th Century A.D., Ealhswith...</td>\n",+ " <td id=\"T_50ac6_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",+ " <td id=\"T_50ac6_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f219daed8a0>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/html": [+ "\n",+ " <div style='\n",+ " text-align: center;\n",+ " font-size: 16px;\n",+ " font-weight: bold;\n",+ " color: #555;\n",+ " margin: 10px 0;'>\n",+ " ... 295 more rows not displayed ...\n",+ " </div>\n",+ " "+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/plain": [+ "60.0"+ ]+ },+ "execution_count": 10,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "optimized_aggregator2 = optimized_aggregator.deepcopy()\n",+ "optimized_aggregator2.temperature = 0.7\n",+ "\n",+ "evaluate(optimized_aggregator2)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 7) Conclusion.\n",+ "\n",+ "Normally, we like to release notebooks with pre-computed caches and to inspect the prompts with `gpt3.inspect_history` to explore the behavior of optimization. See the intro notebook (or any of the Colab notebooks on the README) for such annotated examples!\n",+ "\n",+ "To keep the current release super quick, Omar will extend this notebook into an annotated version if there's significant interest."+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "py310",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.10.13"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/examples/qa/hotpot/multihop_finetune.ipynb b/examples/qa/hotpot/multihop_finetune.ipynbindex 3e4642158..ec67735e6 100644--- a/examples/qa/hotpot/multihop_finetune.ipynb+++ b/examples/qa/hotpot/multihop_finetune.ipynb@@ -156,7 +156,7 @@" ensemble = [prog for *_, prog in basicmh_bs.candidate_programs[:4]]\n","\n"," for idx, prog in enumerate(ensemble):\n",- " # prog.save(f'multihop_llama213b_{idx}.json')\n",+ " # prog.save(f'checkpoints/multihop_llama213b_{idx}.json')\n"," pass"]},@@ -171,7 +171,7 @@"\n"," for idx in range(4):\n"," prog = BasicMH()\n",- " prog.load(f'multihop_llama213b_{idx}.json')\n",+ " prog.load(f'checkpoints/multihop_llama213b_{idx}.json')\n"," ensemble.append(prog)"]},diff --git a/examples/qa/hotpot/multihop_llama213b_0.json b/tests/multihop_llama213b_0.jsonsimilarity index 100%rename from examples/qa/hotpot/multihop_llama213b_0.jsonrename to tests/multihop_llama213b_0.jsondiff --git a/examples/qa/hotpot/multihop_llama213b_1.json b/tests/multihop_llama213b_1.jsonsimilarity index 100%rename from examples/qa/hotpot/multihop_llama213b_1.jsonrename to tests/multihop_llama213b_1.jsondiff --git a/examples/qa/hotpot/multihop_llama213b_2.json b/tests/multihop_llama213b_2.jsonsimilarity index 100%rename from examples/qa/hotpot/multihop_llama213b_2.jsonrename to tests/multihop_llama213b_2.jsondiff --git a/examples/qa/hotpot/multihop_llama213b_3.json b/tests/multihop_llama213b_3.jsonsimilarity index 100%rename from examples/qa/hotpot/multihop_llama213b_3.jsonrename to tests/multihop_llama213b_3.json