Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Revert "Add example in intro notebook"

jphillips414a39bd1169
diff --git a/intro.ipynb b/intro.ipynb
index 890874d0b..60bc0ebaf 100644
--- a/intro.ipynb
+++ b/intro.ipynb
@@ -37,16 +37,7 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "c:\\dspy\\venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
- " from .autonotebook import tqdm as notebook_tqdm\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
@@ -149,14 +140,6 @@
"execution_count": 3,
"metadata": {},
"outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "c:\\dspy\\venv\\Lib\\site-packages\\datasets\\table.py:1421: FutureWarning: promote has been superseded by promote_options='default'.\n",
- " table = cls._concat_blocks(blocks, axis=0)\n"
- ]
- },
{
"data": {
"text/plain": [
@@ -230,7 +213,7 @@
"text": [
"Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?\n",
"Answer: English\n",
- "Relevant Wikipedia Titles: {'Restaurant: Impossible', 'Robert Irvine'}\n"
+ "Relevant Wikipedia Titles: {'Robert Irvine', 'Restaurant: Impossible'}\n"
]
}
],
@@ -610,25 +593,18 @@
"execution_count": 15,
"metadata": {},
"outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- ".\\cache\\compiler\n"
- ]
- },
{
"name": "stderr",
"output_type": "stream",
"text": [
- " 55%|█████▌ | 11/20 [00:13<00:10, 1.20s/it]"
+ " 50%|█████ | 10/20 [00:00<00:00, 121.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Bootstrapped 4 full traces after 12 examples in round 0.\n"
+ "Bootstrapped 4 full traces after 11 examples in round 0.\n"
]
},
{
@@ -890,105 +866,96 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 23 / 50 (46.0): 100%|██████████| 50/50 [00:58<00:00, 1.18s/it]"
+ "Average Metric: 22 / 50 (44.0): 100%|██████████| 50/50 [00:00<00:00, 116.45it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 23 / 50 (46.0%)\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n",
- "c:\\dspy\\dspy\\evaluate\\evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
- " df = df.applymap(truncate_cell)\n"
+ "Average Metric: 22 / 50 (44.0%)\n"
]
},
{
"data": {
"text/html": [
"<style type=\"text/css\">\n",
- "#T_d858d th {\n",
+ "#T_46dbe th {\n",
" text-align: left;\n",
"}\n",
- "#T_d858d td {\n",
+ "#T_46dbe td {\n",
" text-align: left;\n",
"}\n",
- "#T_d858d_row0_col0, #T_d858d_row0_col1, #T_d858d_row0_col2, #T_d858d_row0_col3, #T_d858d_row0_col4, #T_d858d_row0_col5, #T_d858d_row1_col0, #T_d858d_row1_col1, #T_d858d_row1_col2, #T_d858d_row1_col3, #T_d858d_row1_col4, #T_d858d_row1_col5, #T_d858d_row2_col0, #T_d858d_row2_col1, #T_d858d_row2_col2, #T_d858d_row2_col3, #T_d858d_row2_col4, #T_d858d_row2_col5, #T_d858d_row3_col0, #T_d858d_row3_col1, #T_d858d_row3_col2, #T_d858d_row3_col3, #T_d858d_row3_col4, #T_d858d_row3_col5, #T_d858d_row4_col0, #T_d858d_row4_col1, #T_d858d_row4_col2, #T_d858d_row4_col3, #T_d858d_row4_col4, #T_d858d_row4_col5 {\n",
+ "#T_46dbe_row0_col0, #T_46dbe_row0_col1, #T_46dbe_row0_col2, #T_46dbe_row0_col3, #T_46dbe_row0_col4, #T_46dbe_row0_col5, #T_46dbe_row1_col0, #T_46dbe_row1_col1, #T_46dbe_row1_col2, #T_46dbe_row1_col3, #T_46dbe_row1_col4, #T_46dbe_row1_col5, #T_46dbe_row2_col0, #T_46dbe_row2_col1, #T_46dbe_row2_col2, #T_46dbe_row2_col3, #T_46dbe_row2_col4, #T_46dbe_row2_col5, #T_46dbe_row3_col0, #T_46dbe_row3_col1, #T_46dbe_row3_col2, #T_46dbe_row3_col3, #T_46dbe_row3_col4, #T_46dbe_row3_col5, #T_46dbe_row4_col0, #T_46dbe_row4_col1, #T_46dbe_row4_col2, #T_46dbe_row4_col3, #T_46dbe_row4_col4, #T_46dbe_row4_col5 {\n",
" text-align: left;\n",
" white-space: pre-wrap;\n",
" word-wrap: break-word;\n",
" max-width: 400px;\n",
"}\n",
"</style>\n",
- "<table id=\"T_d858d\">\n",
+ "<table id=\"T_46dbe\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
- " <th id=\"T_d858d_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
- " <th id=\"T_d858d_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
- " <th id=\"T_d858d_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
- " <th id=\"T_d858d_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
- " <th id=\"T_d858d_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
- " <th id=\"T_d858d_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",
+ " <th id=\"T_46dbe_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
+ " <th id=\"T_46dbe_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
+ " <th id=\"T_46dbe_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
+ " <th id=\"T_46dbe_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
+ " <th id=\"T_46dbe_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
+ " <th id=\"T_46dbe_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th id=\"T_d858d_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
- " <td id=\"T_d858d_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
- " <td id=\"T_d858d_row0_col1\" class=\"data row0 col1\" >no</td>\n",
- " <td id=\"T_d858d_row0_col2\" class=\"data row0 col2\" >{'Qionghai', 'Cangzhou'}</td>\n",
- " <td id=\"T_d858d_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
- " <td id=\"T_d858d_row0_col4\" class=\"data row0 col4\" >No</td>\n",
- " <td id=\"T_d858d_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_46dbe_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_46dbe_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
+ " <td id=\"T_46dbe_row0_col1\" class=\"data row0 col1\" >no</td>\n",
+ " <td id=\"T_46dbe_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",
+ " <td id=\"T_46dbe_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
+ " <td id=\"T_46dbe_row0_col4\" class=\"data row0 col4\" >No</td>\n",
+ " <td id=\"T_46dbe_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_d858d_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
- " <td id=\"T_d858d_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
- " <td id=\"T_d858d_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
- " <td id=\"T_d858d_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
- " <td id=\"T_d858d_row1_col3\" class=\"data row1 col3\" >['2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was...</td>\n",
- " <td id=\"T_d858d_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",
- " <td id=\"T_d858d_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_46dbe_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_46dbe_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
+ " <td id=\"T_46dbe_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
+ " <td id=\"T_46dbe_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
+ " <td id=\"T_46dbe_row1_col3\" class=\"data row1 col3\" >['2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was...</td>\n",
+ " <td id=\"T_46dbe_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",
+ " <td id=\"T_46dbe_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_d858d_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
- " <td id=\"T_d858d_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
- " <td id=\"T_d858d_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
- " <td id=\"T_d858d_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
- " <td id=\"T_d858d_row2_col3\" class=\"data row2 col3\" >['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",
- " <td id=\"T_d858d_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
- " <td id=\"T_d858d_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_46dbe_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_46dbe_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
+ " <td id=\"T_46dbe_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
+ " <td id=\"T_46dbe_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
+ " <td id=\"T_46dbe_row2_col3\" class=\"data row2 col3\" >['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",
+ " <td id=\"T_46dbe_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
+ " <td id=\"T_46dbe_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_d858d_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
- " <td id=\"T_d858d_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
- " <td id=\"T_d858d_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
- " <td id=\"T_d858d_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",
- " <td id=\"T_d858d_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
- " <td id=\"T_d858d_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
- " <td id=\"T_d858d_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_46dbe_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_46dbe_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
+ " <td id=\"T_46dbe_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
+ " <td id=\"T_46dbe_row3_col2\" class=\"data row3 col2\" >{'Crichton Castle', 'Crichton Collegiate Church'}</td>\n",
+ " <td id=\"T_46dbe_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
+ " <td id=\"T_46dbe_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
+ " <td id=\"T_46dbe_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_d858d_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
- " <td id=\"T_d858d_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
- " <td id=\"T_d858d_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
- " <td id=\"T_d858d_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",
- " <td id=\"T_d858d_row4_col3\" class=\"data row4 col3\" >[\"Æthelweard of East Anglia | Æthelweard (died 854) was a 9th-century king of East Anglia, the long-lived Anglo-Saxon kingdom which today includes the English counties...</td>\n",
- " <td id=\"T_d858d_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
- " <td id=\"T_d858d_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_46dbe_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_46dbe_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
+ " <td id=\"T_46dbe_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
+ " <td id=\"T_46dbe_row4_col2\" class=\"data row4 col2\" >{'Æthelweard (son of Alfred)', 'Ealhswith'}</td>\n",
+ " <td id=\"T_46dbe_row4_col3\" class=\"data row4 col3\" >[\"Æthelweard of East Anglia | Æthelweard (died 854) was a 9th-century king of East Anglia, the long-lived Anglo-Saxon kingdom which today includes the English counties...</td>\n",
+ " <td id=\"T_46dbe_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
+ " <td id=\"T_46dbe_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
- "<pandas.io.formats.style.Styler at 0x2187fe0bb90>"
+ "<pandas.io.formats.style.Styler at 0x7fbb387dcee0>"
]
},
"metadata": {},
@@ -1018,7 +985,7 @@
{
"data": {
"text/plain": [
- "46.0"
+ "44.0"
]
},
"execution_count": 19,
@@ -1057,7 +1024,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 13 / 50 (26.0): 100%|██████████| 50/50 [00:00<00:00, 1013.32it/s]"
+ "Average Metric: 13 / 50 (26.0): 100%|██████████| 50/50 [00:00<00:00, 671.76it/s]"
]
},
{
@@ -1071,91 +1038,89 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "\n",
- "c:\\dspy\\dspy\\evaluate\\evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
- " df = df.applymap(truncate_cell)\n"
+ "\n"
]
},
{
"data": {
"text/html": [
"<style type=\"text/css\">\n",
- "#T_83185 th {\n",
+ "#T_3ac3d th {\n",
" text-align: left;\n",
"}\n",
- "#T_83185 td {\n",
+ "#T_3ac3d td {\n",
" text-align: left;\n",
"}\n",
- "#T_83185_row0_col0, #T_83185_row0_col1, #T_83185_row0_col2, #T_83185_row0_col3, #T_83185_row0_col4, #T_83185_row0_col5, #T_83185_row1_col0, #T_83185_row1_col1, #T_83185_row1_col2, #T_83185_row1_col3, #T_83185_row1_col4, #T_83185_row1_col5, #T_83185_row2_col0, #T_83185_row2_col1, #T_83185_row2_col2, #T_83185_row2_col3, #T_83185_row2_col4, #T_83185_row2_col5, #T_83185_row3_col0, #T_83185_row3_col1, #T_83185_row3_col2, #T_83185_row3_col3, #T_83185_row3_col4, #T_83185_row3_col5, #T_83185_row4_col0, #T_83185_row4_col1, #T_83185_row4_col2, #T_83185_row4_col3, #T_83185_row4_col4, #T_83185_row4_col5 {\n",
+ "#T_3ac3d_row0_col0, #T_3ac3d_row0_col1, #T_3ac3d_row0_col2, #T_3ac3d_row0_col3, #T_3ac3d_row0_col4, #T_3ac3d_row0_col5, #T_3ac3d_row1_col0, #T_3ac3d_row1_col1, #T_3ac3d_row1_col2, #T_3ac3d_row1_col3, #T_3ac3d_row1_col4, #T_3ac3d_row1_col5, #T_3ac3d_row2_col0, #T_3ac3d_row2_col1, #T_3ac3d_row2_col2, #T_3ac3d_row2_col3, #T_3ac3d_row2_col4, #T_3ac3d_row2_col5, #T_3ac3d_row3_col0, #T_3ac3d_row3_col1, #T_3ac3d_row3_col2, #T_3ac3d_row3_col3, #T_3ac3d_row3_col4, #T_3ac3d_row3_col5, #T_3ac3d_row4_col0, #T_3ac3d_row4_col1, #T_3ac3d_row4_col2, #T_3ac3d_row4_col3, #T_3ac3d_row4_col4, #T_3ac3d_row4_col5 {\n",
" text-align: left;\n",
" white-space: pre-wrap;\n",
" word-wrap: break-word;\n",
" max-width: 400px;\n",
"}\n",
"</style>\n",
- "<table id=\"T_83185\">\n",
+ "<table id=\"T_3ac3d\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
- " <th id=\"T_83185_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
- " <th id=\"T_83185_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
- " <th id=\"T_83185_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
- " <th id=\"T_83185_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
- " <th id=\"T_83185_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
- " <th id=\"T_83185_level0_col5\" class=\"col_heading level0 col5\" >gold_passages_retrieved</th>\n",
+ " <th id=\"T_3ac3d_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
+ " <th id=\"T_3ac3d_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
+ " <th id=\"T_3ac3d_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
+ " <th id=\"T_3ac3d_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
+ " <th id=\"T_3ac3d_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
+ " <th id=\"T_3ac3d_level0_col5\" class=\"col_heading level0 col5\" >gold_passages_retrieved</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th id=\"T_83185_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
- " <td id=\"T_83185_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
- " <td id=\"T_83185_row0_col1\" class=\"data row0 col1\" >no</td>\n",
- " <td id=\"T_83185_row0_col2\" class=\"data row0 col2\" >{'Qionghai', 'Cangzhou'}</td>\n",
- " <td id=\"T_83185_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
- " <td id=\"T_83185_row0_col4\" class=\"data row0 col4\" >No</td>\n",
- " <td id=\"T_83185_row0_col5\" class=\"data row0 col5\" >False</td>\n",
+ " <th id=\"T_3ac3d_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_3ac3d_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
+ " <td id=\"T_3ac3d_row0_col1\" class=\"data row0 col1\" >no</td>\n",
+ " <td id=\"T_3ac3d_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",
+ " <td id=\"T_3ac3d_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
+ " <td id=\"T_3ac3d_row0_col4\" class=\"data row0 col4\" >No</td>\n",
+ " <td id=\"T_3ac3d_row0_col5\" class=\"data row0 col5\" >❌ [False]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_83185_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
- " <td id=\"T_83185_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
- " <td id=\"T_83185_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
- " <td id=\"T_83185_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
- " <td id=\"T_83185_row1_col3\" class=\"data row1 col3\" >['2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was...</td>\n",
- " <td id=\"T_83185_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",
- " <td id=\"T_83185_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3ac3d_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_3ac3d_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
+ " <td id=\"T_3ac3d_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
+ " <td id=\"T_3ac3d_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
+ " <td id=\"T_3ac3d_row1_col3\" class=\"data row1 col3\" >['2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was...</td>\n",
+ " <td id=\"T_3ac3d_row1_col4\" class=\"data row1 col4\" >National Hockey League</td>\n",
+ " <td id=\"T_3ac3d_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_83185_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
- " <td id=\"T_83185_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
- " <td id=\"T_83185_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
- " <td id=\"T_83185_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
- " <td id=\"T_83185_row2_col3\" class=\"data row2 col3\" >['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",
- " <td id=\"T_83185_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
- " <td id=\"T_83185_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3ac3d_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_3ac3d_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
+ " <td id=\"T_3ac3d_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
+ " <td id=\"T_3ac3d_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
+ " <td id=\"T_3ac3d_row2_col3\" class=\"data row2 col3\" >['Steve Yzerman | Stephen Gregory \"Steve\" Yzerman ( ; born May 9, 1965) is a Canadian retired professional ice hockey player and current general manager...</td>\n",
+ " <td id=\"T_3ac3d_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
+ " <td id=\"T_3ac3d_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_83185_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
- " <td id=\"T_83185_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
- " <td id=\"T_83185_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
- " <td id=\"T_83185_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",
- " <td id=\"T_83185_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
- " <td id=\"T_83185_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
- " <td id=\"T_83185_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3ac3d_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_3ac3d_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
+ " <td id=\"T_3ac3d_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
+ " <td id=\"T_3ac3d_row3_col2\" class=\"data row3 col2\" >{'Crichton Castle', 'Crichton Collegiate Church'}</td>\n",
+ " <td id=\"T_3ac3d_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
+ " <td id=\"T_3ac3d_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
+ " <td id=\"T_3ac3d_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_83185_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
- " <td id=\"T_83185_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
- " <td id=\"T_83185_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
- " <td id=\"T_83185_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",
- " <td id=\"T_83185_row4_col3\" class=\"data row4 col3\" >[\"Æthelweard of East Anglia | Æthelweard (died 854) was a 9th-century king of East Anglia, the long-lived Anglo-Saxon kingdom which today includes the English counties...</td>\n",
- " <td id=\"T_83185_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
- " <td id=\"T_83185_row4_col5\" class=\"data row4 col5\" >False</td>\n",
+ " <th id=\"T_3ac3d_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_3ac3d_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
+ " <td id=\"T_3ac3d_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
+ " <td id=\"T_3ac3d_row4_col2\" class=\"data row4 col2\" >{'Æthelweard (son of Alfred)', 'Ealhswith'}</td>\n",
+ " <td id=\"T_3ac3d_row4_col3\" class=\"data row4 col3\" >[\"Æthelweard of East Anglia | Æthelweard (died 854) was a 9th-century king of East Anglia, the long-lived Anglo-Saxon kingdom which today includes the English counties...</td>\n",
+ " <td id=\"T_3ac3d_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
+ " <td id=\"T_3ac3d_row4_col5\" class=\"data row4 col5\" >❌ [False]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
- "<pandas.io.formats.style.Styler at 0x2187fe50a10>"
+ "<pandas.io.formats.style.Styler at 0x7fbb387909d0>"
]
},
"metadata": {},
@@ -1314,7 +1279,7 @@
"text": [
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"Predicted Answer: five\n",
- "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', 'David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas....', 'Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd...', 'Kinnairdy Castle | Kinnairdy Castle is a tower house, having five storeys and a garret, two miles south of Aberchirder, Aberdeenshire, Scotland. The alternative name is Old Kinnairdy....', 'Gregory of Gaeta | Gregory was the Duke of Gaeta from 963 until his death. He was the second son of Docibilis II of Gaeta and his wife Orania. He succeeded his brother John II, who had left only daugh...']\n"
+ "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', 'The Boleyn Inheritance | The Boleyn Inheritance is a novel by British author Philippa Gregory which was first published in 2006. It is a direct sequel to her previous novel \"The Other Boleyn Girl,\" an...', 'Gregory of Gaeta | Gregory was the Duke of Gaeta from 963 until his death. He was the second son of Docibilis II of Gaeta and his wife Orania. He succeeded his brother John II, who had left only daugh...', 'Kinnairdy Castle | Kinnairdy Castle is a tower house, having five storeys and a garret, two miles south of Aberchirder, Aberdeenshire, Scotland. The alternative name is Old Kinnairdy....', 'Kinnaird Head | Kinnaird Head (Scottish Gaelic: \"An Ceann Àrd\" , \"high headland\") is a headland projecting into the North Sea, within the town of Fraserburgh, Aberdeenshire on the east coast of Scotla...', 'Kinnaird Castle, Brechin | Kinnaird Castle is a 15th-century castle in Angus, Scotland. The castle has been home to the Carnegie family, the Earl of Southesk, for more than 600 years....']\n"
]
}
],
@@ -1372,9 +1337,9 @@
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
- "Reasoning: Let's think step by step in order to\u001b[32m find the answer to this question. First, we need to gather information about David Gregory and the castle he inherited. Then, we can search for any available information about the castle's architecture or historical records that mention the number of storeys.\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m find the answer to this question. First, we need to find information about David Gregory and the castle he inherited. Then, we can search for details about the castle's architecture or any historical records that mention the number of storeys.\n",
"\n",
- "Query: \"David Gregory castle inherited storeys\"\u001b[0m\n",
+ "Query: \"David Gregory castle inheritance\"\u001b[0m\n",
"\n",
"\n",
"\n",
@@ -1400,14 +1365,14 @@
"\n",
"Context:\n",
"[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory's use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\n",
- "[2] «David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas.»\n",
- "[3] «Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd Lord Castlestewart, eldest son of Andrew Stewart (1580-1629) the third Lord Ochiltree, 1st Lord Castlestewart who came from Scotland during the plantation and established the nearby town of Stewartstown. Andrew Stewart junior acquired the land of Ballokevan from Robert Stewart between 1610 and 1619 and built his castle overlooking Roughan Lough. It is a small square castle, three storeys high with a central tower 20 ft square, flanked by thick rounded towers at each corner.»\n",
+ "[2] «The Boleyn Inheritance | The Boleyn Inheritance is a novel by British author Philippa Gregory which was first published in 2006. It is a direct sequel to her previous novel \"The Other Boleyn Girl,\" and one of the additions to her six-part series on the Tudor royals. (The other titles in the series are \"The Constant Princess,\" \"The Queen's Fool,\" \"The Virgin's Lover,and The Other Queen.\") * The novel is told through the first-person narratives of – Anne of Cleves, Katherine Howard, and Jane Boleyn, who was mentioned in \"The Other Boleyn Girl.\" It covers a period from 1539 until 1542 and chronicles the fourth and fifth marriages of King Henry VIII of England.»\n",
+ "[3] «Gregory of Gaeta | Gregory was the Duke of Gaeta from 963 until his death. He was the second son of Docibilis II of Gaeta and his wife Orania. He succeeded his brother John II, who had left only daughters. Gregory rapidly depleted the \"publicum\" (public land) of the Duchy of Gaeta by doling it out to family members as grants. Gregory disappears from the records in 964 and was succeeded by his younger brother Marinus of Fondi over the heads of his three sons. It is possible that there was an internal power struggle between factions of the Docibilan family and that Gregory was forced out. On the other hand, perhaps he died and his sons fought a losing battle for their inheritance to Gaeta.»\n",
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
- "Reasoning: Let's think step by step in order to\u001b[32m find the answer to the question. First, we need to identify which castle David Gregory inherited. Then, we can look for information about the castle's structure, specifically the number of storeys.\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the query. We know that David Gregory inherited Kinnairdy Castle, so we need to find information about the castle and its characteristics.\n",
"\n",
- "Query: \"David Gregory inherited castle Kinnairdy storeys\"\u001b[0m\n",
+ "Query: \"Kinnairdy Castle number of storeys\"\u001b[0m\n",
"\n",
"\n",
"\n",
@@ -1433,10 +1398,11 @@
"\n",
"Context:\n",
"[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory's use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\n",
- "[2] «David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas.»\n",
- "[3] «Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd Lord Castlestewart, eldest son of Andrew Stewart (1580-1629) the third Lord Ochiltree, 1st Lord Castlestewart who came from Scotland during the plantation and established the nearby town of Stewartstown. Andrew Stewart junior acquired the land of Ballokevan from Robert Stewart between 1610 and 1619 and built his castle overlooking Roughan Lough. It is a small square castle, three storeys high with a central tower 20 ft square, flanked by thick rounded towers at each corner.»\n",
+ "[2] «The Boleyn Inheritance | The Boleyn Inheritance is a novel by British author Philippa Gregory which was first published in 2006. It is a direct sequel to her previous novel \"The Other Boleyn Girl,\" and one of the additions to her six-part series on the Tudor royals. (The other titles in the series are \"The Constant Princess,\" \"The Queen's Fool,\" \"The Virgin's Lover,and The Other Queen.\") * The novel is told through the first-person narratives of – Anne of Cleves, Katherine Howard, and Jane Boleyn, who was mentioned in \"The Other Boleyn Girl.\" It covers a period from 1539 until 1542 and chronicles the fourth and fifth marriages of King Henry VIII of England.»\n",
+ "[3] «Gregory of Gaeta | Gregory was the Duke of Gaeta from 963 until his death. He was the second son of Docibilis II of Gaeta and his wife Orania. He succeeded his brother John II, who had left only daughters. Gregory rapidly depleted the \"publicum\" (public land) of the Duchy of Gaeta by doling it out to family members as grants. Gregory disappears from the records in 964 and was succeeded by his younger brother Marinus of Fondi over the heads of his three sons. It is possible that there was an internal power struggle between factions of the Docibilan family and that Gregory was forced out. On the other hand, perhaps he died and his sons fought a losing battle for their inheritance to Gaeta.»\n",
"[4] «Kinnairdy Castle | Kinnairdy Castle is a tower house, having five storeys and a garret, two miles south of Aberchirder, Aberdeenshire, Scotland. The alternative name is Old Kinnairdy.»\n",
- "[5] «Gregory of Gaeta | Gregory was the Duke of Gaeta from 963 until his death. He was the second son of Docibilis II of Gaeta and his wife Orania. He succeeded his brother John II, who had left only daughters. Gregory rapidly depleted the \"publicum\" (public land) of the Duchy of Gaeta by doling it out to family members as grants. Gregory disappears from the records in 964 and was succeeded by his younger brother Marinus of Fondi over the heads of his three sons. It is possible that there was an internal power struggle between factions of the Docibilan family and that Gregory was forced out. On the other hand, perhaps he died and his sons fought a losing battle for their inheritance to Gaeta.»\n",
+ "[5] «Kinnaird Head | Kinnaird Head (Scottish Gaelic: \"An Ceann Àrd\" , \"high headland\") is a headland projecting into the North Sea, within the town of Fraserburgh, Aberdeenshire on the east coast of Scotland. The 16th-century Kinnaird Castle was converted in 1787 for use as the Kinnaird Head Lighthouse, the first lighthouse in Scotland to be lit by the Commissioners of Northern Lights. Kinnaird Castle and the nearby Winetower were described by W. Douglas Simpson as two of the nine castles of the Knuckle, referring to the rocky headland of north-east Aberdeenshire. Both buildings are category A listed buildings.»\n",
+ "[6] «Kinnaird Castle, Brechin | Kinnaird Castle is a 15th-century castle in Angus, Scotland. The castle has been home to the Carnegie family, the Earl of Southesk, for more than 600 years.»\n",
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
@@ -1503,21 +1469,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- " 90%|█████████ | 18/20 [01:02<00:06, 3.48s/it]"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 4 full traces after 19 examples in round 0.\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n"
+ "100%|██████████| 20/20 [00:00<00:00, 64.11it/s]\n"
]
}
],
@@ -1545,125 +1497,108 @@
"metadata": {},
"outputs": [],
"source": [
- "# Let's utilize the Evaluate return_outputs option to see the full differences between the uncopiled and compiled programs.\n",
- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5, return_outputs=True)"
+ "uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved, display=False)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
- "outputs": [],
- "source": [
- "uncompiled_baleen_retrieval_score, uncompiled_results = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved, display=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 29,
- "metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 29 / 50 (58.0): 100%|██████████| 50/50 [03:14<00:00, 3.88s/it]"
+ "Average Metric: 30 / 50 (60.0): 100%|██████████| 50/50 [00:00<00:00, 54.98it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 29 / 50 (58.0%)\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n"
+ "Average Metric: 30 / 50 (60.0%)\n"
]
},
{
"data": {
"text/html": [
"<style type=\"text/css\">\n",
- "#T_e3874 th {\n",
+ "#T_3bd2b th {\n",
" text-align: left;\n",
"}\n",
- "#T_e3874 td {\n",
+ "#T_3bd2b td {\n",
" text-align: left;\n",
"}\n",
- "#T_e3874_row0_col0, #T_e3874_row0_col1, #T_e3874_row0_col2, #T_e3874_row0_col3, #T_e3874_row0_col4, #T_e3874_row0_col5, #T_e3874_row1_col0, #T_e3874_row1_col1, #T_e3874_row1_col2, #T_e3874_row1_col3, #T_e3874_row1_col4, #T_e3874_row1_col5, #T_e3874_row2_col0, #T_e3874_row2_col1, #T_e3874_row2_col2, #T_e3874_row2_col3, #T_e3874_row2_col4, #T_e3874_row2_col5, #T_e3874_row3_col0, #T_e3874_row3_col1, #T_e3874_row3_col2, #T_e3874_row3_col3, #T_e3874_row3_col4, #T_e3874_row3_col5, #T_e3874_row4_col0, #T_e3874_row4_col1, #T_e3874_row4_col2, #T_e3874_row4_col3, #T_e3874_row4_col4, #T_e3874_row4_col5 {\n",
+ "#T_3bd2b_row0_col0, #T_3bd2b_row0_col1, #T_3bd2b_row0_col2, #T_3bd2b_row0_col3, #T_3bd2b_row0_col4, #T_3bd2b_row0_col5, #T_3bd2b_row1_col0, #T_3bd2b_row1_col1, #T_3bd2b_row1_col2, #T_3bd2b_row1_col3, #T_3bd2b_row1_col4, #T_3bd2b_row1_col5, #T_3bd2b_row2_col0, #T_3bd2b_row2_col1, #T_3bd2b_row2_col2, #T_3bd2b_row2_col3, #T_3bd2b_row2_col4, #T_3bd2b_row2_col5, #T_3bd2b_row3_col0, #T_3bd2b_row3_col1, #T_3bd2b_row3_col2, #T_3bd2b_row3_col3, #T_3bd2b_row3_col4, #T_3bd2b_row3_col5, #T_3bd2b_row4_col0, #T_3bd2b_row4_col1, #T_3bd2b_row4_col2, #T_3bd2b_row4_col3, #T_3bd2b_row4_col4, #T_3bd2b_row4_col5 {\n",
" text-align: left;\n",
" white-space: pre-wrap;\n",
" word-wrap: break-word;\n",
" max-width: 400px;\n",
"}\n",
"</style>\n",
- "<table id=\"T_e3874\">\n",
+ "<table id=\"T_3bd2b\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
- " <th id=\"T_e3874_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
- " <th id=\"T_e3874_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
- " <th id=\"T_e3874_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
- " <th id=\"T_e3874_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
- " <th id=\"T_e3874_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
- " <th id=\"T_e3874_level0_col5\" class=\"col_heading level0 col5\" >gold_passages_retrieved</th>\n",
+ " <th id=\"T_3bd2b_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
+ " <th id=\"T_3bd2b_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
+ " <th id=\"T_3bd2b_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
+ " <th id=\"T_3bd2b_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
+ " <th id=\"T_3bd2b_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
+ " <th id=\"T_3bd2b_level0_col5\" class=\"col_heading level0 col5\" >gold_passages_retrieved</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th id=\"T_e3874_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
- " <td id=\"T_e3874_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
- " <td id=\"T_e3874_row0_col1\" class=\"data row0 col1\" >no</td>\n",
- " <td id=\"T_e3874_row0_col2\" class=\"data row0 col2\" >{'Qionghai', 'Cangzhou'}</td>\n",
- " <td id=\"T_e3874_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
- " <td id=\"T_e3874_row0_col4\" class=\"data row0 col4\" >No</td>\n",
- " <td id=\"T_e3874_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3bd2b_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_3bd2b_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
+ " <td id=\"T_3bd2b_row0_col1\" class=\"data row0 col1\" >no</td>\n",
+ " <td id=\"T_3bd2b_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",
+ " <td id=\"T_3bd2b_row0_col3\" class=\"data row0 col3\" >['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area...</td>\n",
+ " <td id=\"T_3bd2b_row0_col4\" class=\"data row0 col4\" >No</td>\n",
+ " <td id=\"T_3bd2b_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_e3874_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
- " <td id=\"T_e3874_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
- " <td id=\"T_e3874_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
- " <td id=\"T_e3874_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
- " <td id=\"T_e3874_row1_col3\" class=\"data row1 col3\" >[\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National...</td>\n",
- " <td id=\"T_e3874_row1_col4\" class=\"data row1 col4\" >2017 NHL Expansion Draft</td>\n",
- " <td id=\"T_e3874_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3bd2b_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_3bd2b_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
+ " <td id=\"T_3bd2b_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
+ " <td id=\"T_3bd2b_row1_col2\" class=\"data row1 col2\" >{'2017 NHL Expansion Draft', '2017–18 Pittsburgh Penguins season'}</td>\n",
+ " <td id=\"T_3bd2b_row1_col3\" class=\"data row1 col3\" >[\"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to...</td>\n",
+ " <td id=\"T_3bd2b_row1_col4\" class=\"data row1 col4\" >National Hockey League (NHL)</td>\n",
+ " <td id=\"T_3bd2b_row1_col5\" class=\"data row1 col5\" >❌ [False]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_e3874_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
- " <td id=\"T_e3874_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
- " <td id=\"T_e3874_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
- " <td id=\"T_e3874_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
- " <td id=\"T_e3874_row2_col3\" class=\"data row2 col3\" >['List of Tampa Bay Lightning general managers | The Tampa Bay Lightning are an American professional ice hockey team based in Tampa, Florida. They play...</td>\n",
- " <td id=\"T_e3874_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
- " <td id=\"T_e3874_row2_col5\" class=\"data row2 col5\" >False</td>\n",
+ " <th id=\"T_3bd2b_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_3bd2b_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
+ " <td id=\"T_3bd2b_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
+ " <td id=\"T_3bd2b_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
+ " <td id=\"T_3bd2b_row2_col3\" class=\"data row2 col3\" >['List of Tampa Bay Lightning general managers | The Tampa Bay Lightning are an American professional ice hockey team based in Tampa, Florida. They play...</td>\n",
+ " <td id=\"T_3bd2b_row2_col4\" class=\"data row2 col4\" >Steve Yzerman</td>\n",
+ " <td id=\"T_3bd2b_row2_col5\" class=\"data row2 col5\" >❌ [False]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_e3874_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
- " <td id=\"T_e3874_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
- " <td id=\"T_e3874_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
- " <td id=\"T_e3874_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",
- " <td id=\"T_e3874_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
- " <td id=\"T_e3874_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
- " <td id=\"T_e3874_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
+ " <th id=\"T_3bd2b_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_3bd2b_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
+ " <td id=\"T_3bd2b_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
+ " <td id=\"T_3bd2b_row3_col2\" class=\"data row3 col2\" >{'Crichton Castle', 'Crichton Collegiate Church'}</td>\n",
+ " <td id=\"T_3bd2b_row3_col3\" class=\"data row3 col3\" >[\"Crichton Collegiate Church | Crichton Collegiate Church is situated about 0.6 mi south west of the hamlet of Crichton in Midlothian, Scotland. Crichton itself is...</td>\n",
+ " <td id=\"T_3bd2b_row3_col4\" class=\"data row3 col4\" >River Tyne</td>\n",
+ " <td id=\"T_3bd2b_row3_col5\" class=\"data row3 col5\" >✔️ [True]</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_e3874_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
- " <td id=\"T_e3874_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
- " <td id=\"T_e3874_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
- " <td id=\"T_e3874_row4_col2\" class=\"data row4 col2\" >{'Ealhswith', 'Æthelweard (son of Alfred)'}</td>\n",
- " <td id=\"T_e3874_row4_col3\" class=\"data row4 col3\" >[\"Æthelweard of East Anglia | Æthelweard (died 854) was a 9th-century king of East Anglia, the long-lived Anglo-Saxon kingdom which today includes the English counties...</td>\n",
- " <td id=\"T_e3874_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
- " <td id=\"T_e3874_row4_col5\" class=\"data row4 col5\" >False</td>\n",
+ " <th id=\"T_3bd2b_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_3bd2b_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
+ " <td id=\"T_3bd2b_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
+ " <td id=\"T_3bd2b_row4_col2\" class=\"data row4 col2\" >{'Æthelweard (son of Alfred)', 'Ealhswith'}</td>\n",
+ " <td id=\"T_3bd2b_row4_col3\" class=\"data row4 col3\" >['Æthelweard (son of Alfred) | Æthelweard (d. 920 or 922) was the younger son of King Alfred the Great and Ealhswith.', 'Æthelred the Unready |...</td>\n",
+ " <td id=\"T_3bd2b_row4_col4\" class=\"data row4 col4\" >King Alfred the Great</td>\n",
+ " <td id=\"T_3bd2b_row4_col5\" class=\"data row4 col5\" >❌ [False]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
- "<pandas.io.formats.style.Styler at 0x21880047390>"
+ "<pandas.io.formats.style.Styler at 0x7fbaf80fab80>"
]
},
"metadata": {},
@@ -1692,12 +1627,12 @@
}
],
"source": [
- "compiled_baleen_retrieval_score, compiled_results = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)"
+ "compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)"
]
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
@@ -1706,7 +1641,7 @@
"text": [
"## Retrieval Score for RAG: 26.0\n",
"## Retrieval Score for uncompiled Baleen: 36.0\n",
- "## Retrieval Score for compiled Baleen: 58.0\n"
+ "## Retrieval Score for compiled Baleen: 60.0\n"
]
}
],
@@ -1716,173 +1651,6 @@
"print(f\"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}\")"
]
},
- {
- "cell_type": "code",
- "execution_count": 36,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "<div>\n",
- "<style scoped>\n",
- " .dataframe tbody tr th:only-of-type {\n",
- " vertical-align: middle;\n",
- " }\n",
- "\n",
- " .dataframe tbody tr th {\n",
- " vertical-align: top;\n",
- " }\n",
- "\n",
- " .dataframe thead th {\n",
- " text-align: right;\n",
- " }\n",
- "</style>\n",
- "<table border=\"1\" class=\"dataframe\">\n",
- " <thead>\n",
- " <tr style=\"text-align: right;\">\n",
- " <th></th>\n",
- " <th>Question</th>\n",
- " <th>Uncompiled Answer</th>\n",
- " <th>Compiled Answer</th>\n",
- " </tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr>\n",
- " <th>0</th>\n",
- " <td>Are both Cangzhou and Qionghai in the Hebei pr...</td>\n",
- " <td>No.</td>\n",
- " <td>No</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>1</th>\n",
- " <td>Who conducts the draft in which Marc-Andre Fle...</td>\n",
- " <td>National Hockey League (NHL)</td>\n",
- " <td>2017 NHL Expansion Draft</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>2</th>\n",
- " <td>The Wings entered a new era, following the ret...</td>\n",
- " <td>Steve Yzerman</td>\n",
- " <td>Steve Yzerman</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>3</th>\n",
- " <td>What river is near the Crichton Collegiate Chu...</td>\n",
- " <td>River Esk</td>\n",
- " <td>River Tyne</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>4</th>\n",
- " <td>In the 10th Century A.D. Ealhswith had a son c...</td>\n",
- " <td>Alfred the Great</td>\n",
- " <td>King Alfred the Great</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>5</th>\n",
- " <td>The Newark Airport Exchange is at the northern...</td>\n",
- " <td>Port Authority of New York and New Jersey</td>\n",
- " <td>Port Authority of New York and New Jersey</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>6</th>\n",
- " <td>Where did an event take place resulting in a w...</td>\n",
- " <td>Lima</td>\n",
- " <td>Copa America</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>7</th>\n",
- " <td>Are both Chico Municipal Airport and William R...</td>\n",
- " <td>No.</td>\n",
- " <td>No</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>8</th>\n",
- " <td>In which Maine county is Fort Pownall located?</td>\n",
- " <td>Waldo</td>\n",
- " <td>Waldo County</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>9</th>\n",
- " <td>Which 90s rock band has more recently reformed...</td>\n",
- " <td>The Afghan Whigs</td>\n",
- " <td>The Afghan Whigs</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>10</th>\n",
- " <td>What year did the mountain known in Italian as...</td>\n",
- " <td>79 AD</td>\n",
- " <td>79</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>11</th>\n",
- " <td>Is the 72nd field brigade part of the oldest o...</td>\n",
- " <td>Unknown</td>\n",
- " <td>oldest</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>12</th>\n",
- " <td>Was Stanislaw Kiszka paid for his services by ...</td>\n",
- " <td>Unknown</td>\n",
- " <td>No</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>13</th>\n",
- " <td>Which film director is younger, Del Lord or Wa...</td>\n",
- " <td>Wang Xiaoshuai</td>\n",
- " <td>Wang Xiaoshuai</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>14</th>\n",
- " <td>Lord North Street has a resident in which form...</td>\n",
- " <td>Jonathan Aitken</td>\n",
- " <td>Jonathan Aitken</td>\n",
- " </tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</div>"
- ],
- "text/plain": [
- " Question Uncompiled Answer Compiled Answer\n",
- "0 Are both Cangzhou and Qionghai in the Hebei pr... No. No\n",
- "1 Who conducts the draft in which Marc-Andre Fle... National Hockey League (NHL) 2017 NHL Expansion Draft\n",
- "2 The Wings entered a new era, following the ret... Steve Yzerman Steve Yzerman\n",
- "3 What river is near the Crichton Collegiate Chu... River Esk River Tyne\n",
- "4 In the 10th Century A.D. Ealhswith had a son c... Alfred the Great King Alfred the Great\n",
- "5 The Newark Airport Exchange is at the northern... Port Authority of New York and New Jersey Port Authority of New York and New Jersey\n",
- "6 Where did an event take place resulting in a w... Lima Copa America\n",
- "7 Are both Chico Municipal Airport and William R... No. No\n",
- "8 In which Maine county is Fort Pownall located? Waldo Waldo County\n",
- "9 Which 90s rock band has more recently reformed... The Afghan Whigs The Afghan Whigs\n",
- "10 What year did the mountain known in Italian as... 79 AD 79\n",
- "11 Is the 72nd field brigade part of the oldest o... Unknown oldest\n",
- "12 Was Stanislaw Kiszka paid for his services by ... Unknown No\n",
- "13 Which film director is younger, Del Lord or Wa... Wang Xiaoshuai Wang Xiaoshuai\n",
- "14 Lord North Street has a resident in which form... Jonathan Aitken Jonathan Aitken"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "import pandas as pd\n",
- "from IPython.display import display\n",
- "data = []\n",
- "for uncompiled, compiled in zip(uncompiled_results, compiled_results):\n",
- " question = uncompiled[0]['question']\n",
- " uncompiled_answer = uncompiled[1].answer\n",
- " compiled_answer = compiled[1].answer\n",
- " data.append((question, uncompiled_answer, compiled_answer))\n",
- "\n",
- "# Create DataFrame\n",
- "df = pd.DataFrame(data, columns=['Question', 'Uncompiled Answer', 'Compiled Answer'])\n",
- "\n",
- "# Display the DataFrame\n",
- "pd.set_option('display.max_colwidth', 50)\n",
- "display(df.head(15))"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -1903,7 +1671,7 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 30,
"metadata": {},
"outputs": [
{
@@ -1932,16 +1700,6 @@
"\n",
"Context: N/A\n",
"\n",
- "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
- "\n",
- "Reasoning: Let's think step by step in order to determine which magazine has published articles by Scott Shaw. We can start by searching for Scott Shaw's name in the archives of Tae Kwon Do Times and Southwest Art magazines.\n",
- "\n",
- "Query: \"Scott Shaw\" site:tkdtimes.com OR site:southwestart.com\n",
- "\n",
- "---\n",
- "\n",
- "Context: N/A\n",
- "\n",
"Question: In what year was the club founded that played Manchester City in the 1972 FA Charity Shield\n",
"\n",
"Reasoning: Let's think step by step in order to produce the query. We know that the FA Charity Shield is an annual football match played in England between the winners of the previous season's Premier League and FA Cup. In this case, we are looking for the year when Manchester City played against a specific club in the 1972 FA Charity Shield. To find this information, we can search for the history of the FA Charity Shield and the teams that participated in the 1972 edition.\n",
@@ -1974,9 +1732,9 @@
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
- "Reasoning: Let's think step by step in order to\u001b[32m produce the query. We can search for information about David Gregory and the castle he inherited to find out how many storeys it has.\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the query. We need to find information about the castle that David Gregory inherited and determine the number of storeys it has.\n",
"\n",
- "Query: \"David Gregory castle inherited storeys\"\u001b[0m\n",
+ "Query: \"Castle inherited by David Gregory number of storeys\"\u001b[0m\n",
"\n",
"\n",
"\n",
@@ -2001,18 +1759,6 @@
"---\n",
"\n",
"Context:\n",
- "[1] «Scott Shaw (artist) | Scott Shaw (often spelled Scott Shaw!) is a United States cartoonist and animator, and historian of comics. Among Scott's comic-book work is Hanna-Barbera's \"The Flintstones\" (for Marvel Comics and Harvey Comics), \"Captain Carrot and His Amazing Zoo Crew\" (for DC Comics), and \"Simpsons Comics\" (for Bongo Comics). He was also the first artist for Archie Comics' \"Sonic the Hedgehog\" comic book series.»\n",
- "[2] «Scott Shaw | Scott Shaw (born September 23, 1958) is an American actor, author, film director, film producer, journalist, martial artist, musician, photographer, and professor.»\n",
- "\n",
- "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
- "\n",
- "Reasoning: Let's think step by step in order to produce the query. We know that Scott Shaw is a multi-talented individual with various interests and skills. He has worked in different fields such as comics, animation, and martial arts. To find out which magazine has published articles by Scott Shaw, we need to search for information related to his writings or contributions in these magazines.\n",
- "\n",
- "Query: \"Scott Shaw articles Tae Kwon Do Times\" OR \"Scott Shaw articles Southwest Art\"\n",
- "\n",
- "---\n",
- "\n",
- "Context:\n",
"[1] «1972 FA Charity Shield | The 1972 FA Charity Shield was contested between Manchester City and Aston Villa.»\n",
"[2] «1971 FA Charity Shield | The 1971 FA Charity Shield was a football match between Leicester City and Liverpool at Filbert Street on Saturday 7 August 1971.»\n",
"\n",
@@ -2050,12 +1796,12 @@
"\n",
"Context:\n",
"[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory's use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\n",
- "[2] «David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas.»\n",
- "[3] «Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd Lord Castlestewart, eldest son of Andrew Stewart (1580-1629) the third Lord Ochiltree, 1st Lord Castlestewart who came from Scotland during the plantation and established the nearby town of Stewartstown. Andrew Stewart junior acquired the land of Ballokevan from Robert Stewart between 1610 and 1619 and built his castle overlooking Roughan Lough. It is a small square castle, three storeys high with a central tower 20 ft square, flanked by thick rounded towers at each corner.»\n",
+ "[2] «Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 to 1006. In December 999, and again on February 2, 1002, he reinstituted and confirmed the possessions of the abbey and monks of Monte Cassino in Ascoli. In 1004, he fortified and expanded the castle of Dragonara on the Fortore. He gave it three circular towers and one square one. He also strengthened Lucera.»\n",
+ "[3] «Gregory Parsloe-Parsloe | Sir Gregory Parsloe-Parsloe, 7th Baronet is a fictional character from the Blandings stories of P. G. Wodehouse. The seventh Baronet, who resides at Matchingham Hall, he is the son of the Very Reverend Dean Parsloe-Parsloe and is the rival and enemy of Lord Emsworth, master of Blandings Castle.»\n",
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
- "Reasoning: Let's think step by step in order to\u001b[32m produce the query. We know that David Gregory inherited Kinnairdy Castle and that he had three mathematics professor children. To find out how many storeys are in the castle that David Gregory inherited, we need to search for information about Kinnairdy Castle.\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the query. We know that David Gregory inherited Kinnairdy Castle and that he had three of his twenty-nine children become mathematics professors. To find out how many storeys are in the castle, we need to search for information about Kinnairdy Castle.\n",
"\n",
"Query: \"Kinnairdy Castle number of storeys\"\u001b[0m\n",
"\n",
@@ -2069,38 +1815,41 @@
"\n",
"---\n",
"\n",
- "Question: Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?\n",
- "Answer: Rosario Dawson\n",
- "\n",
"Question: The Organisation that allows a community to influence their operation or use and to enjoy the benefits arisingwas founded in what year?\n",
"Answer: 2010\n",
"\n",
- "Question: Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?\n",
- "Answer: Aleem Sarwar Dar\n",
+ "Question: On the coast of what ocean is the birthplace of Diogal Sakho?\n",
+ "Answer: Atlantic\n",
"\n",
- "Question: Tombstone stared an actor born May 17, 1955 known as who?\n",
- "Answer: Bill Paxton\n",
+ "Question: Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?\n",
+ "Answer: Buena Vista Distribution\n",
+ "\n",
+ "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
+ "Answer: Tae Kwon Do Times\n",
+ "\n",
+ "Question: Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?\n",
+ "Answer: Rosario Dawson\n",
+ "\n",
+ "Question: Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\" ?\n",
+ "Answer: Kerry Condon\n",
+ "\n",
+ "Question: which American actor was Candace Kita guest starred with\n",
+ "Answer: Bill Murray\n",
"\n",
"Question: \"Everything Has Changed\" is a song from an album released under which record label ?\n",
"Answer: Big Machine Records\n",
"\n",
- "Question: The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?\n",
- "Answer: 1950\n",
- "\n",
- "Question: Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where?\n",
- "Answer: space\n",
+ "Question: Tombstone stared an actor born May 17, 1955 known as who?\n",
+ "Answer: Bill Paxton\n",
"\n",
- "Question: At My Window was released by which American singer-songwriter?\n",
- "Answer: John Townes Van Zandt\n",
+ "Question: Which of these publications was most recently published, Who Put the Bomp or Self?\n",
+ "Answer: Self\n",
"\n",
"Question: What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?\n",
"Answer: Operation Citadel\n",
"\n",
- "Question: which American actor was Candace Kita guest starred with\n",
- "Answer: Bill Murray\n",
- "\n",
- "Question: Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\" ?\n",
- "Answer: Kerry Condon\n",
+ "Question: Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where?\n",
+ "Answer: space\n",
"\n",
"Question: Having the combination of excellent foot speed and bat speed helped Eric Davis, create what kind of outfield for the Los Angeles Dodgers?\n",
"Answer: \"Outfield of Dreams\"\n",
@@ -2120,19 +1869,6 @@
"---\n",
"\n",
"Context:\n",
- "[1] «Scott Shaw (artist) | Scott Shaw (often spelled Scott Shaw!) is a United States cartoonist and animator, and historian of comics. Among Scott's comic-book work is Hanna-Barbera's \"The Flintstones\" (for Marvel Comics and Harvey Comics), \"Captain Carrot and His Amazing Zoo Crew\" (for DC Comics), and \"Simpsons Comics\" (for Bongo Comics). He was also the first artist for Archie Comics' \"Sonic the Hedgehog\" comic book series.»\n",
- "[2] «Scott Shaw | Scott Shaw (born September 23, 1958) is an American actor, author, film director, film producer, journalist, martial artist, musician, photographer, and professor.»\n",
- "[3] «Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of taekwondo, and is published in the United States of America. While the title suggests that it focuses on taekwondo exclusively, the magazine also covers other Korean martial arts. \"Tae Kwon Do Times\" has published articles by a wide range of authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van Schuyver.»\n",
- "\n",
- "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
- "\n",
- "Reasoning: Let's think step by step in order to produce the answer. We know that Scott Shaw has published articles in Tae Kwon Do Times according to [3]. We don't have any information about Scott Shaw publishing in Southwest Art. Therefore, the answer is Tae Kwon Do Times.\n",
- "\n",
- "Answer: Tae Kwon Do Times\n",
- "\n",
- "---\n",
- "\n",
- "Context:\n",
"[1] «1972 FA Charity Shield | The 1972 FA Charity Shield was contested between Manchester City and Aston Villa.»\n",
"[2] «1971 FA Charity Shield | The 1971 FA Charity Shield was a football match between Leicester City and Liverpool at Filbert Street on Saturday 7 August 1971.»\n",
"[3] «1896–97 Aston Villa F.C. season | The 1896-87 season was Aston Villa's ninth season in the Football League since being one of its 12 founding members in 1888.»\n",
@@ -2176,17 +1912,17 @@
"\n",
"Context:\n",
"[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory's use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\n",
- "[2] «David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas.»\n",
- "[3] «Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd Lord Castlestewart, eldest son of Andrew Stewart (1580-1629) the third Lord Ochiltree, 1st Lord Castlestewart who came from Scotland during the plantation and established the nearby town of Stewartstown. Andrew Stewart junior acquired the land of Ballokevan from Robert Stewart between 1610 and 1619 and built his castle overlooking Roughan Lough. It is a small square castle, three storeys high with a central tower 20 ft square, flanked by thick rounded towers at each corner.»\n",
+ "[2] «Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 to 1006. In December 999, and again on February 2, 1002, he reinstituted and confirmed the possessions of the abbey and monks of Monte Cassino in Ascoli. In 1004, he fortified and expanded the castle of Dragonara on the Fortore. He gave it three circular towers and one square one. He also strengthened Lucera.»\n",
+ "[3] «Gregory Parsloe-Parsloe | Sir Gregory Parsloe-Parsloe, 7th Baronet is a fictional character from the Blandings stories of P. G. Wodehouse. The seventh Baronet, who resides at Matchingham Hall, he is the son of the Very Reverend Dean Parsloe-Parsloe and is the rival and enemy of Lord Emsworth, master of Blandings Castle.»\n",
"[4] «Kinnairdy Castle | Kinnairdy Castle is a tower house, having five storeys and a garret, two miles south of Aberchirder, Aberdeenshire, Scotland. The alternative name is Old Kinnairdy.»\n",
"[5] «Kinnaird Head | Kinnaird Head (Scottish Gaelic: \"An Ceann Àrd\" , \"high headland\") is a headland projecting into the North Sea, within the town of Fraserburgh, Aberdeenshire on the east coast of Scotland. The 16th-century Kinnaird Castle was converted in 1787 for use as the Kinnaird Head Lighthouse, the first lighthouse in Scotland to be lit by the Commissioners of Northern Lights. Kinnaird Castle and the nearby Winetower were described by W. Douglas Simpson as two of the nine castles of the Knuckle, referring to the rocky headland of north-east Aberdeenshire. Both buildings are category A listed buildings.»\n",
"[6] «Kinnaird Castle, Brechin | Kinnaird Castle is a 15th-century castle in Angus, Scotland. The castle has been home to the Carnegie family, the Earl of Southesk, for more than 600 years.»\n",
"\n",
"Question: How many storeys are in the castle that David Gregory inherited?\n",
"\n",
- "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We know that David Gregory inherited Kinnairdy Castle. According to [4], Kinnairdy Castle is a tower house with five storeys and a garret. Therefore, the castle that David Gregory inherited has five storeys.\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We know that David Gregory inherited Kinnairdy Castle. According to the context, Kinnairdy Castle is a tower house with five storeys and a garret.\n",
"\n",
- "Answer: five\u001b[0m\n",
+ "Answer: Five\u001b[0m\n",
"\n",
"\n",
"\n"
@@ -2215,7 +1951,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.4"
+ "version": "3.9.17"
},
"orig_nbformat": 4
},