DSPy: The framework for programming—not prompting—language models
Python
OAI data kinda working
Isaac Miller967dcc52364a
diff --git a/dsp/modules/openai.py b/dsp/modules/openai.pyindex 3e3935437..4028f57cf 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/openai.py@@ -256,12 +256,12 @@ class OpenAIModel(GPT3, FinetuneableLM):# TODO: support OAI wandb integrationdef start_remote_training(self, **kwargs) -> str:assert self.fine_tuning_file_ids["train"] is not None, "You must load data before starting training"-+ hyperparameters = kwargs.get("hyperparameters", {})job = openai.fine_tuning.jobs.create(training_file=self.fine_tuning_file_ids["train"],model=self.kwargs["model"],seed=kwargs.get("seed", 0),- hyperparameters=hparams,+ hyperparameters=hyperparameters,validation_file=self.fine_tuning_file_ids.get("val", None),)self.fine_tuning_job_id = job.id@@ -331,7 +331,7 @@ class OpenAIModel(GPT3, FinetuneableLM):raise RuntimeError("Job not completed yet, cannot retrieve model")def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':- # Maybe validate data here+ # Maybe validate data, hparams hereself._load_data(train_path, val_path)if method != "SFT":raise NotImplementedError("Only SFT finetuning is supported at the moment.")diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynbindex 9e9ee6209..5312a06b0 100644--- a/examples/finetuning/openai_finetuning_hpqa.ipynb+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb@@ -199,211 +199,46 @@},{"cell_type": "code",- "execution_count": 17,+ "execution_count": 10,"metadata": {},"outputs": [{"name": "stderr","output_type": "stream","text": [- "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 3905.97it/s]"+ "Average Metric: 62 / 100 (62.0): 100%|██████████| 100/100 [00:00<00:00, 351.07it/s]\n"]},{"name": "stdout","output_type": "stream","text": [- "[(Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- ")), {'context': [], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",- " rationale='determine whether both Cangzhou and Qionghai are in the Hebei province of China. First, we need to confirm the geographical locations of Cangzhou and Qionghai. This will involve searching for maps or geographical information about these cities to verify their respective provinces.',\n",- " search_query='\"Cangzhou Qionghai locations provinces China\"'\n",- ")), (Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- ")), {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\"], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",- " rationale='determine the geographical locations of Cangzhou and Qionghai. The context indicates that Cangzhou is in Hebei province, while it does not provide information about Qionghai. To answer the question, we need to find out whether Qionghai is also located in Hebei province. This requires a search that confirms the province of Qionghai.',\n",- " search_query='\"Qionghai location province\"'\n",- ")), (Predict(StringSignature(context, question -> rationale, answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- ")), {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\", 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.', \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\", \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"], 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'}, Prediction(\n",- " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",- " answer='no'\n",- "))]\n",- "[(Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- ")), {'context': [], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",- " rationale='determine who conducted the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. First, we need to identify the NHL draft process and find out who was responsible for selecting players for the team during that specific draft.',\n",- " search_query='\"Marc-Andre Fleury draft Vegas Golden Knights 2017-18 conductor\"'\n",- ")), (Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- ")), {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\"], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",- " rationale='produce the search_query. We need to identify who is responsible for conducting the NHL Expansion Draft in which Marc-André Fleury was selected by the Vegas Golden Knights. The context provides information about the 2017 NHL Expansion Draft, which hints that it is organized by the National Hockey League (NHL). Therefore, we can search for details about the expansion draft and its organizers.',\n",- " search_query='\"NHL Expansion Draft 2017 organizer\"'\n",- ")), (Predict(StringSignature(context, question -> rationale, answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- ")), {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\", \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\", \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"], 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'}, Prediction(\n",- " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",- " answer='National Hockey League'\n",- "))]\n"+ "186\n"]},{"name": "stderr","output_type": "stream","text": [- "\n"+ "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 343.97it/s]\n"]},{- "data": {- "text/plain": [- "[{'example': Example({'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?', 'answer': 'no'}) (input_keys={'question'}),\n",- " 'prediction': Prediction(\n",- " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",- " answer='no',\n",- " context=['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.', \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\", \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\", 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.', \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\", \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"]\n",- " ),\n",- " 'trace': [(Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- " )),\n",- " {'context': [],\n",- " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",- " Prediction(\n",- " rationale='determine whether both Cangzhou and Qionghai are in the Hebei province of China. First, we need to confirm the geographical locations of Cangzhou and Qionghai. This will involve searching for maps or geographical information about these cities to verify their respective provinces.',\n",- " search_query='\"Cangzhou Qionghai locations provinces China\"'\n",- " )),\n",- " (Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- " )),\n",- " {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.',\n",- " \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\",\n",- " \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\"],\n",- " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",- " Prediction(\n",- " rationale='determine the geographical locations of Cangzhou and Qionghai. The context indicates that Cangzhou is in Hebei province, while it does not provide information about Qionghai. To answer the question, we need to find out whether Qionghai is also located in Hebei province. This requires a search that confirms the province of Qionghai.',\n",- " search_query='\"Qionghai location province\"'\n",- " )),\n",- " (Predict(StringSignature(context, question -> rationale, answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- " )),\n",- " {'context': ['Cangzhou | Cangzhou () is a prefecture-level city in eastern Hebei province, People\\'s Republic of China. At the 2010 census, Cangzhou\\'s built-up (\"or metro\") area made of Yunhe, Xinhua districts and Cang County largely being conurbated had a population of 1,205,814 inhabitants, while the prefecture-level administrative unit in total has a population of 7,134,062. It lies approximately 90 km from the major port city of Tianjin, and 180 km from Beijing.',\n",- " \"Nantong | Nantong (; former names: Nan-t'ung, Nantung, Tongzhou, or Tungchow; Qihai dialect: ] ) is a prefecture-level city in Jiangsu province, China. Located on the northern bank of the Yangtze River, near the river mouth. Nantong is a vital river port bordering Yancheng to the north, Taizhou to the west, Suzhou and Shanghai to the south across the river, and the East China Sea to the east.\",\n",- " \"Dongguang County | Dongguang County () is a county under the jurisdiction of Cangzhou City, in southeastern Hebei province, People's Republic of China, bordering Shandong to the southeast.\",\n",- " 'Qionghai | Qionghai () is one of the seven county-level cities of Hainan province, China. Although called a \"city\", Qionghai refers to a large land area in Hainan - an area which was once a county. Within this area is the main city, Qionghai City. It is located in the east of the island at the mouth of the Wanquan River, 86 km from the provincial capital of Haikou. It has an area of 1692 km2 and in 2010, it had a population of 483,217.',\n",- " \"Qionghai Bo'ao Airport | Qionghai Bo'ao Airport (IATA: BAR, ICAO: ZJQH) is an airport serving the city of Qionghai in Hainan Province, China. The airport received approval from the national government in January 2013, and was opened on 17 March 2016 after three years of construction.\",\n",- " \"Qionglai City | Qionglai () is a county-level city under the jurisdiction of Chengdu, the capital of Sichuan province, People's Republic of China. It is located around 60 km from downtown Chengdu. The city is located on the western edge of the Sichuan Basin and in the foothills of the Qionglai Mountains that bound the basin from the west, and is bordered by the prefecture-level city of Ya'an to the west.\"],\n",- " 'question': 'Are both Cangzhou and Qionghai in the Hebei province of China?'},\n",- " Prediction(\n",- " rationale='determine if both Cangzhou and Qionghai are in the Hebei province of China. The context mentions that Cangzhou is a city in eastern Hebei province. However, it does not mention Qionghai being in Hebei; instead, it states that Qionghai is in Hainan province. Since they are located in different provinces, we can conclude that both are not in Hebei province.',\n",- " answer='no'\n",- " ))],\n",- " 'score': True,\n",- " 'round': 0},\n",- " {'example': Example({'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?', 'answer': 'National Hockey League'}) (input_keys={'question'}),\n",- " 'prediction': Prediction(\n",- " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",- " answer='National Hockey League',\n",- " context=[\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\", '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.', \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\", \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\", \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"]\n",- " ),\n",- " 'trace': [(Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- " )),\n",- " {'context': [],\n",- " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",- " Prediction(\n",- " rationale='determine who conducted the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season. First, we need to identify the NHL draft process and find out who was responsible for selecting players for the team during that specific draft.',\n",- " search_query='\"Marc-Andre Fleury draft Vegas Golden Knights 2017-18 conductor\"'\n",- " )),\n",- " (Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- " )),\n",- " {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\",\n",- " '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.',\n",- " \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\"],\n",- " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",- " Prediction(\n",- " rationale='produce the search_query. We need to identify who is responsible for conducting the NHL Expansion Draft in which Marc-André Fleury was selected by the Vegas Golden Knights. The context provides information about the 2017 NHL Expansion Draft, which hints that it is organized by the National Hockey League (NHL). Therefore, we can search for details about the expansion draft and its organizers.',\n",- " search_query='\"NHL Expansion Draft 2017 organizer\"'\n",- " )),\n",- " (Predict(StringSignature(context, question -> rationale, answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- " )),\n",- " {'context': [\"Marc-André Fleury | Marc-André Fleury (born November 28, 1984) is a French-Canadian professional ice hockey goaltender playing for the Vegas Golden Knights of the National Hockey League (NHL). Drafted out of the Quebec Major Junior Hockey League (QMJHL) first overall by the Pittsburgh Penguins in the 2003 NHL Entry Draft, Fleury played major junior for four seasons with the Cape Breton Screaming Eagles, earning both the Mike Bossy Trophy as the league's top prospect and the Telus Cup as the top defensive player in 2003. He joined the Penguins in 2003–04 and has won three Stanley Cup championships with the team in 2009, 2016, and 2017. Internationally, Fleury has represented Canada twice as a junior, winning back-to-back silver medals at the World Junior Championships in 2003 and 2004. He won a gold medal with Team Canada at the 2010 Winter Olympics in Vancouver.\",\n",- " '2017–18 Pittsburgh Penguins season | The 2017–18 Pittsburgh Penguins season will be the 51st season for the National Hockey League ice hockey team that was established on June 5, 1967. They will enter the season as two-time defending Stanley Cup champions. It will be the first season since the 2002–03 season in which the team will play without goaltender Marc-Andre Fleury, who was drafted to the Vegas Golden Knights in the expansion draft.',\n",- " \"2017 NHL Expansion Draft | The 2017 NHL Expansion Draft was an expansion draft conducted by the National Hockey League on June 18–20, 2017 to fill the roster of the league's expansion team for the 2017–18 season, the Vegas Golden Knights. The team's selections were announced on June 21 during the NHL Awards ceremony at T-Mobile Arena.\",\n",- " \"1967 NHL Expansion Draft | The 1967 NHL Expansion Draft was held on June 6, 1967, in the ballroom of the Queen Elizabeth Hotel in Montreal, Quebec, Canada. The draft took place to fill the rosters of the league's six expansion teams for the 1967–68 season: the California Seals, Los Angeles Kings, Minnesota North Stars, Philadelphia Flyers, Pittsburgh Penguins and the St. Louis Blues.\",\n",- " \"Potential National Hockey League expansion | The National Hockey League (NHL) has undergone several rounds of expansion and other organizational changes during its 100-year history to reach its current thirty-one teams: twenty-four in the United States, and seven in Canada. The last time the NHL added expansion teams was in 2016 , when the league approved the Vegas Golden Knights for play in the 2017–18 season. The league's most recent relocation was in 2011, when the former Atlanta Thrashers relocated to become the modern Winnipeg Jets.\"],\n",- " 'question': 'Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?'},\n",- " Prediction(\n",- " rationale='identify who conducts the draft for the NHL expansion team, the Vegas Golden Knights. The context mentions the 2017 NHL Expansion Draft, which was conducted by the National Hockey League. Therefore, it can be concluded that the draft was conducted by the NHL.',\n",- " answer='National Hockey League'\n",- " ))],\n",- " 'score': True,\n",- " 'round': 0}]"- ]- },- "execution_count": 17,- "metadata": {},- "output_type": "execute_result"+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "168\n"+ ]}],"source": ["from collections import Counter\n","from dspy.teleprompt.finetune_teleprompter import bootstrap_data_multiple_rounds, DataCollectionCallback, build_prompt_completion_data_from_trace\n","from typing import Callable\n",+ "import ujson\n","\n","samples = 100\n","\n",- "\n",- "\n","callback = DataCollectionCallback(num_correct=1, max_attempts=1)\n","\n","dspy.settings.configure(experimental=True)\n",@@ -416,14 +251,25 @@"}\n","basicmh_bs = BasicMH(**program_params)\n","basicmh_bs.load(\"basicmh_3_3_6.json\")\n",- "\n",- "results = bootstrap_data_multiple_rounds(basicmh_bs, devset[:samples], metric, num_rounds=1, **dc_kwargs)\n",- "for result in results[:2]:\n",- " print(result[\"trace\"])\n",- "\n",- "traces = [build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in results]\n",- "\n",- "traces[:2]"+ "# NOTE: It looks like all data, not just success is saved\n",+ "\n",+ "trainset_data = bootstrap_data_multiple_rounds(basicmh_bs, trainset[:samples], metric, num_rounds=1, **dc_kwargs)\n",+ "trainset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in trainset_data if result[\"score\"]], [])\n",+ "trainset_data = lm.format_data_for_vanilla_finetuning(trainset_data)\n",+ "print(len(trainset_data))\n",+ "ts_json = ujson.dumps(trainset_data)\n",+ "with open(\"trainset_data.jsonl\", \"w\") as f:\n",+ " for line in trainset_data:\n",+ " f.write(ujson.dumps(line) + \"\\n\")\n",+ "\n",+ "devset_data = bootstrap_data_multiple_rounds(basicmh_bs, devset[:int(samples)], metric, num_rounds=1, **dc_kwargs)\n",+ "devset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in devset_data if result[\"score\"]], [])\n",+ "devset_data = lm.format_data_for_vanilla_finetuning(devset_data)\n",+ "print(len(devset_data))\n",+ "ds_json = ujson.dumps(devset_data)\n",+ "with open(\"devset_data.jsonl\", \"w\") as f:\n",+ " for line in devset_data:\n",+ " f.write(ujson.dumps(line) + \"\\n\")"]},{@@ -431,6 +277,15 @@"execution_count": 11,"metadata": {},"outputs": [],+ "source": [+ "lm.start_training(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],"source": ["# from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n","# ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\n",@@ -439,7 +294,7 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -466,7 +321,7 @@},{"cell_type": "code",- "execution_count": 13,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -481,7 +336,7 @@},{"cell_type": "code",- "execution_count": 14,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -500,38 +355,9 @@},{"cell_type": "code",- "execution_count": 15,+ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 133 / 300 (44.3): 100%|██████████| 300/300 [00:00<00:00, 314.72it/s]\n",- "Average Metric: 158 / 300 (52.7): 100%|██████████| 300/300 [00:00<00:00, 332.45it/s]\n",- "Average Metric: 177 / 300 (59.0): 100%|██████████| 300/300 [01:35<00:00, 3.14it/s]\n",- "Average Metric: 180 / 300 (60.0): 100%|██████████| 300/300 [01:31<00:00, 3.27it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N=100 and up to 1 attempts for each example with one model for all predictors. Tested on first 300 of devset.\n",- "Non-finetuned model: 44.33\n",- "Non-finetuned bootstrapped model: 52.67\n",- "Finetuned model: 59.0\n",- "Finetuned model with bootstrapping: 60.0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"- ]- }- ],+ "outputs": [],"source": ["TEST_SIZE = 300\n","\n",