Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

WIP Lm

Isaac Miller3ca59b769947
diff --git a/dsp/modules/openai.py b/dsp/modules/openai.py
index dd268b7dd..3e3935437 100644
--- a/dsp/modules/openai.py
+++ b/dsp/modules/openai.py
@@ -1,4 +1,5 @@
from copy import deepcopy
+import time
from typing import Any, Optional, Literal, Union
import openai
@@ -147,6 +148,7 @@ def backoff_hdlr(details):
"{kwargs}".format(**details),
)
+FinetuningMethod = Literal["SFT", "Contrastive"]
class OpenAIModel(GPT3, FinetuneableLM):
"""Wrapper around specifically the OpenAI API to finetune.
@@ -252,7 +254,7 @@ class OpenAIModel(GPT3, FinetuneableLM):
self.fine_tuning_file_ids[name] = file.id
# TODO: support OAI wandb integration
- def start_training(self, **kwargs) -> str:
+ def start_remote_training(self, **kwargs) -> str:
assert self.fine_tuning_file_ids["train"] is not None, "You must load data before starting training"
job = openai.fine_tuning.jobs.create(
@@ -327,3 +329,21 @@ class OpenAIModel(GPT3, FinetuneableLM):
return self_copy
else:
raise RuntimeError("Job not completed yet, cannot retrieve model")
+
+ def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':
+ # Maybe validate data here
+ self._load_data(train_path, val_path)
+ if method != "SFT":
+ raise NotImplementedError("Only SFT finetuning is supported at the moment.")
+ self.start_remote_training(**kwargs)
+ self.wait_for_training()
+
+ # Return a future that can be used to check the status of the training
+
+ def wait_for_training(self):
+ # Wait for the future to complete
+ while not self.check_training_status():
+ time.sleep(60)
+
+ def check_status(self) -> Optional[str]:
+ pass
diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.py
index 5856857fb..f2553b60e 100644
--- a/dspy/teleprompt/finetune_teleprompter.py
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@ -141,7 +141,7 @@ def build_prompt_completion_data_from_trace(
# add the predictor_name key as well as the lm_kwargs.
data_dict = dict(prompt=prompt, completion=completion)
- if program:
+ if program and id(pred) in pred_id_to_name:
pred_name = pred_id_to_name[id(pred)]
data_dict['predictor_name'] = pred_name
if record_lm_kwargs:
diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynb
index e56e93f51..9e9ee6209 100644
--- a/examples/finetuning/openai_finetuning_hpqa.ipynb
+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb
@@ -2,18 +2,9 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 1,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "The autoreload extension is already loaded. To reload it, use:\n",
- " %reload_ext autoreload\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
@@ -21,9 +12,20 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
+ " warnings.warn(\n",
+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ }
+ ],
"source": [
"import sys\n",
"import os\n",
@@ -47,7 +49,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -68,7 +70,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -104,7 +106,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -122,7 +124,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -132,7 +134,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -159,7 +161,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -169,7 +171,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -197,20 +199,205 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 355.76it/s]\n"
+ "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 3905.97it/s]"
]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[(Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ ")), {'context': [], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",
+ " rationale='determine whether both Cangzhou and Qionghai are in the Hebei province of China. First, we need to confirm the geographical locations of Cangzhou and Qionghai. This will involve searching for maps or geographical information about these cities to verify their respective provinces.',\n",
+ " search_query='\"Cangzhou Qionghai locations provinces China\"'\n",
+ ")), (Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ ")), {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\"], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",
+ " rationale='determine the geographical locations of Cangzhou and Qionghai. The context indicates that Cangzhou is in Hebei province, while it does not provide information about Qionghai. To answer the question, we need to find out whether Qionghai is also located in Hebei province. This requires a search that confirms the province of Qionghai.',\n",
+ " search_query='\"Qionghai location province\"'\n",
+ ")), (Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ ")), {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\", 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.', \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\", \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",
+ " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",
+ " answer='no'\n",
+ "))]\n",
+ "[(Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ ")), {'context': [], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",
+ " rationale='determine who conducted the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. First, we need to identify the NHL draft process and find out who was responsible for selecting players for the team during that specific draft.',\n",
+ " search_query='\"Marc-Andre Fleury draft Vegas Golden Knights 2017-18 conductor\"'\n",
+ ")), (Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ ")), {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\"], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",
+ " rationale='produce the search_query. We need to identify who is responsible for conducting the NHL Expansion Draft in which Marc-André Fleury was selected by the Vegas Golden Knights. The context provides information about the 2017 NHL Expansion Draft, which hints that it is organized by the National Hockey League (NHL). Therefore, we can search for details about the expansion draft and its organizers.',\n",
+ " search_query='\"NHL Expansion Draft 2017 organizer\"'\n",
+ ")), (Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ ")), {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\", \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\", \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",
+ " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",
+ " answer='National Hockey League'\n",
+ "))]\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[{'example': Example({'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?', 'answer': 'no'}) (input_keys={'question'}),\n",
+ " 'prediction': Prediction(\n",
+ " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",
+ " answer='no',\n",
+ " context=['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\", 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.', \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\", \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"]\n",
+ " ),\n",
+ " 'trace': [(Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ " )),\n",
+ " {'context': [],\n",
+ " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",
+ " Prediction(\n",
+ " rationale='determine whether both Cangzhou and Qionghai are in the Hebei province of China. First, we need to confirm the geographical locations of Cangzhou and Qionghai. This will involve searching for maps or geographical information about these cities to verify their respective provinces.',\n",
+ " search_query='\"Cangzhou Qionghai locations provinces China\"'\n",
+ " )),\n",
+ " (Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ " )),\n",
+ " {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.',\n",
+ " \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\",\n",
+ " \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\"],\n",
+ " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",
+ " Prediction(\n",
+ " rationale='determine the geographical locations of Cangzhou and Qionghai. The context indicates that Cangzhou is in Hebei province, while it does not provide information about Qionghai. To answer the question, we need to find out whether Qionghai is also located in Hebei province. This requires a search that confirms the province of Qionghai.',\n",
+ " search_query='\"Qionghai location province\"'\n",
+ " )),\n",
+ " (Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ " )),\n",
+ " {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.',\n",
+ " \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\",\n",
+ " \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\",\n",
+ " 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.',\n",
+ " \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\",\n",
+ " \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"],\n",
+ " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",
+ " Prediction(\n",
+ " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",
+ " answer='no'\n",
+ " ))],\n",
+ " 'score': True,\n",
+ " 'round': 0},\n",
+ " {'example': Example({'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?', 'answer': 'National Hockey League'}) (input_keys={'question'}),\n",
+ " 'prediction': Prediction(\n",
+ " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",
+ " answer='National Hockey League',\n",
+ " context=[\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\", \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\", \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"]\n",
+ " ),\n",
+ " 'trace': [(Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ " )),\n",
+ " {'context': [],\n",
+ " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",
+ " Prediction(\n",
+ " rationale='determine who conducted the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. First, we need to identify the NHL draft process and find out who was responsible for selecting players for the team during that specific draft.',\n",
+ " search_query='\"Marc-Andre Fleury draft Vegas Golden Knights 2017-18 conductor\"'\n",
+ " )),\n",
+ " (Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ " )),\n",
+ " {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\",\n",
+ " '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.',\n",
+ " \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\"],\n",
+ " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",
+ " Prediction(\n",
+ " rationale='produce the search_query. We need to identify who is responsible for conducting the NHL Expansion Draft in which Marc-André Fleury was selected by the Vegas Golden Knights. The context provides information about the 2017 NHL Expansion Draft, which hints that it is organized by the National Hockey League (NHL). Therefore, we can search for details about the expansion draft and its organizers.',\n",
+ " search_query='\"NHL Expansion Draft 2017 organizer\"'\n",
+ " )),\n",
+ " (Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ " )),\n",
+ " {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\",\n",
+ " '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.',\n",
+ " \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\",\n",
+ " \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\",\n",
+ " \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"],\n",
+ " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",
+ " Prediction(\n",
+ " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",
+ " answer='National Hockey League'\n",
+ " ))],\n",
+ " 'score': True,\n",
+ " 'round': 0}]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
}
],
"source": [
"from collections import Counter\n",
- "from dspy.teleprompt.finetune_teleprompter import bootstrap_data_multiple_rounds, DataCollectionCallback\n",
+ "from dspy.teleprompt.finetune_teleprompter import bootstrap_data_multiple_rounds, DataCollectionCallback, build_prompt_completion_data_from_trace\n",
"from typing import Callable\n",
"\n",
"samples = 100\n",
@@ -231,12 +418,17 @@
"basicmh_bs.load(\"basicmh_3_3_6.json\")\n",
"\n",
"results = bootstrap_data_multiple_rounds(basicmh_bs, devset[:samples], metric, num_rounds=1, **dc_kwargs)\n",
- "# results[:2]"
+ "for result in results[:2]:\n",
+ " print(result[\"trace\"])\n",
+ "\n",
+ "traces = [build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in results]\n",
+ "\n",
+ "traces[:2]"
]
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -247,7 +439,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -274,7 +466,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -289,7 +481,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@@ -308,31 +500,35 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 33 / 69 (47.8): 23%|██▎ | 69/300 [00:25<01:03, 3.64it/s]WARNING:dspy.evaluate.evaluate:\u001b[2m2024-08-13T00:56:16.101839Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mReceived SIGINT. Cancelling evaluation.\u001b[0m [\u001b[0m\u001b[1m\u001b[34mdspy.evaluate.evaluate\u001b[0m]\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mevaluate.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m91\u001b[0m\n"
+ "Average Metric: 133 / 300 (44.3): 100%|██████████| 300/300 [00:00<00:00, 314.72it/s]\n",
+ "Average Metric: 158 / 300 (52.7): 100%|██████████| 300/300 [00:00<00:00, 332.45it/s]\n",
+ "Average Metric: 177 / 300 (59.0): 100%|██████████| 300/300 [01:35<00:00, 3.14it/s]\n",
+ "Average Metric: 180 / 300 (60.0): 100%|██████████| 300/300 [01:31<00:00, 3.27it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N=100 and up to 1 attempts for each example with one model for all predictors. Tested on first 300 of devset.\n",
+ "Non-finetuned model: 44.33\n",
+ "Non-finetuned bootstrapped model: 52.67\n",
+ "Finetuned model: 59.0\n",
+ "Finetuned model with bootstrapping: 60.0\n"
]
},
{
- "ename": "KeyboardInterrupt",
- "evalue": "",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[0;32mIn[19], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m TEST_SIZE \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m dspy\u001b[38;5;241m.\u001b[39mcontext(lm\u001b[38;5;241m=\u001b[39mdspy\u001b[38;5;241m.\u001b[39mOpenAI(model\u001b[38;5;241m=\u001b[39mmini, max_tokens\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m500\u001b[39m, temperature\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.0\u001b[39m)):\n\u001b[0;32m----> 3\u001b[0m baseline_eval \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBasicMH\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mprogram_params\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevset\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43mTEST_SIZE\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m bs_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[1;32m 5\u001b[0m bs_ft_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs_ft, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:193\u001b[0m, in \u001b[0;36mEvaluate.__call__\u001b[0;34m(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)\u001b[0m\n\u001b[1;32m 191\u001b[0m reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute_single_thread(wrapped_program, devset, display_progress)\n\u001b[1;32m 192\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 193\u001b[0m reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_multi_thread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[43m \u001b[49m\u001b[43mwrapped_program\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 195\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 196\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_threads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43mdisplay_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 200\u001b[0m dspy\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAverage Metric: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mncorrect\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m / \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mntotal\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mround\u001b[39m(\u001b[38;5;241m100\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39mncorrect\u001b[38;5;250m \u001b[39m\u001b[38;5;241m/\u001b[39m\u001b[38;5;250m \u001b[39mntotal,\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m%)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 202\u001b[0m predicted_devset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m(reordered_devset)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:109\u001b[0m, in \u001b[0;36mEvaluate._execute_multi_thread\u001b[0;34m(self, wrapped_program, devset, num_threads, display_progress)\u001b[0m\n\u001b[1;32m 106\u001b[0m futures \u001b[38;5;241m=\u001b[39m {executor\u001b[38;5;241m.\u001b[39msubmit(cancellable_wrapped_program, idx, arg) \u001b[38;5;28;01mfor\u001b[39;00m idx, arg \u001b[38;5;129;01min\u001b[39;00m devset}\n\u001b[1;32m 107\u001b[0m pbar \u001b[38;5;241m=\u001b[39m tqdm\u001b[38;5;241m.\u001b[39mtqdm(total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(devset), dynamic_ncols\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, disable\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m display_progress)\n\u001b[0;32m--> 109\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m as_completed(futures):\n\u001b[1;32m 110\u001b[0m example_idx, example, prediction, score \u001b[38;5;241m=\u001b[39m future\u001b[38;5;241m.\u001b[39mresult()\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# use the cancelled_job literal to check if the job was cancelled - use \"is\" not \"==\"\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;66;03m# in case the prediction is \"cancelled\" for some reason.\u001b[39;00m\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:245\u001b[0m, in \u001b[0;36mas_completed\u001b[0;34m(fs, timeout)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wait_timeout \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 241\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\n\u001b[1;32m 242\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m (of \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m) futures unfinished\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28mlen\u001b[39m(pending), total_futures))\n\u001b[0;32m--> 245\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwait_timeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m waiter\u001b[38;5;241m.\u001b[39mlock:\n\u001b[1;32m 248\u001b[0m finished \u001b[38;5;241m=\u001b[39m waiter\u001b[38;5;241m.\u001b[39mfinished_futures\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py:574\u001b[0m, in \u001b[0;36mEvent.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 572\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flag\n\u001b[1;32m 573\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m signaled:\n\u001b[0;32m--> 574\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 575\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m signaled\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py:312\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 313\u001b[0m gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:92\u001b[0m, in \u001b[0;36mEvaluate._execute_multi_thread.<locals>.interrupt_handler_manager.<locals>.interrupt_handler\u001b[0;34m(sig, frame)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel_jobs\u001b[38;5;241m.\u001b[39mset()\n\u001b[1;32m 91\u001b[0m dspy\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReceived SIGINT. Cancelling evaluation.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 92\u001b[0m \u001b[43mdefault_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[43msig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mframe\u001b[49m\u001b[43m)\u001b[49m\n",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
]
}
],