DSPy: The framework for programming—not prompting—language models
Python
WIP Future LM + notebook
Isaac Miller8b0dcda95ce3
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 7d9278e1b..4b3fbec2d 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -21,6 +21,7 @@ from .hf_client import Anyscale, HFClientTGI, Togetherfrom .llama import *from .mistral import *from .ollama import *+from .openai import *from .multi_openai import MultiOpenAIfrom .premai import PremAIfrom .pyserini import *diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex 69856b78f..853cfabbb 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -114,7 +114,7 @@ class GPT3(LM):def basic_request(self, prompt: str, **kwargs):raw_kwargs = kwargs- kwargs = {**self.kwargs, **kwargs}+ kwargs = {**(self.kwargs if self.kwargs else {}), **(kwargs if kwargs else {})}if self.model_type == "chat":# caching mechanism requires hashable kwargsmessages = [{"role": "user", "content": prompt}]diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 7f7d064ea..f4aa1d876 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -1,6 +1,9 @@from abc import ABC, abstractmethod+from asyncio import Future+from concurrent.futures import ThreadPoolExecutor+from copy import deepcopyfrom dataclasses import dataclass, field-from typing import Any, Optional+from typing import Any, Literal, Optional@dataclass#(frozen=True) technically should be frozenclass LM(ABC):@@ -172,4 +175,32 @@ class FinetuneableLM(LM, ABC):@abstractmethoddef retrieve_trained_model_client(self) -> LM:+ pass++FinetuningMethod = Literal["SFT", "Contrastive"]+class FinetunableLMFuture(LM, ABC):+ def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLMFuture']:+ future: Future['FinetunableLMFuture'] = Future()++ def run_finetune(original: 'FinetunableLMFuture'):+ try:+ copy = deepcopy(original)+ result = copy.start_training(train_path, val_path, method, **kwargs)+ future.set_result(result)+ except Exception as e:+ future.set_exception(e)++ executor = ThreadPoolExecutor(max_workers=1)++ executor.submit(run_finetune, self)+ executor.shutdown(wait=False)++ return future++ @abstractmethod+ def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':+ pass++ @abstractmethod+ def check_status(self) -> Optional[str]:pass\ No newline at end of filediff --git a/dsp/modules/openai.py b/dsp/modules/openai.pyindex 721d76773..dd268b7dd 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/openai.py@@ -174,8 +174,7 @@ class OpenAIModel(GPT3, FinetuneableLM):fine_tuning_job_id: Optional[str] = None,**kwargs,):- super().__init__(model, api_key, api_provider,- api_base, model_type, system_prompt, **kwargs)+ super().__init__(model, api_key=api_key, api_provider=api_provider, api_base=api_base, model_type=model_type, system_prompt=system_prompt, **kwargs)assert self.provider == "openai", "You must use an OpenAI model with this class."self.fine_tuning_file_ids = fine_tuning_file_idsself.fine_tuning_job_id = fine_tuning_job_id@@ -225,9 +224,9 @@ class OpenAIModel(GPT3, FinetuneableLM):messages = [{"role": "user", "content": item["prompt"]}, {"role": "assistant", "content": item["completion"]}]# NOTE: System prompt is required for the OpenAI API- if messages[0]["role"] != "system":+ if messages[0]["role"] != "system" and self.system_prompt:messages.insert(- 0, {"role": "system", "content": self.system_prompt or "You are a helpful assistant."})+ 0, {"role": "system", "content": self.system_prompt})wrapped_messages = {"messages": messages}return wrapped_messagesreturn list(map(format_single_item, data))diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 78ae04b03..7cba0e252 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -39,6 +39,7 @@ Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal+OpenAIModel = dsp.OpenAIModelLlamaCpp = dsp.LlamaCppBedrock = dsp.Bedrockdiff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex 4b2a5e33f..57ec3a988 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -154,6 +154,7 @@ class Module(BaseModule, metaclass=ProgramMeta):Args:lm: The LM to set."""+ # NOTE: Needs experimental to be truefor pred in self.predictors():pred.lm = lmdiff --git a/dspy/retrieve/retrieve.py b/dspy/retrieve/retrieve.pyindex 05cd66449..70346851a 100644--- a/dspy/retrieve/retrieve.py+++ b/dspy/retrieve/retrieve.py@@ -28,7 +28,7 @@ class Retrieve(Parameter):def reset(self):pass- def dump_state(self):+ def dump_state(self, save_verbose=False):state_keys = ["k"]return {k: getattr(self, k) for k in state_keys}diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.pyindex 07f47d12c..5856857fb 100644--- a/dspy/teleprompt/finetune_teleprompter.py+++ b/dspy/teleprompt/finetune_teleprompter.py@@ -1,8 +1,8 @@-from collections import OrderedDict+from collections import Counter, OrderedDictfrom typing import Any, Callable, Dict, List, Optional, Unionimport dspy-import dspy.logger as logger+from dspy.utils.logging import loggerfrom dspy.evaluate.evaluate import Evaluatefrom dspy.primitives.example import Examplefrom dspy.primitives.program import Program@@ -289,15 +289,17 @@ def bootstrap_data_multiple_rounds(# Ensure that the LM consistency is satisfied, push the global LM to the# predictors if dspy.settings.lm is set- program.assert_lm_consistency()+ # print(program(**dataset[0].inputs()))+ program._assert_lm_consistency()if dspy.settings.lm:- program.set_all_predictor_lms(dspy.settings.lm)+ # This line is breaking everything+ program._set_all_predictor_lms(dspy.settings.lm)# Collect rounds of data collection with different sampling temperaturesdata = []for round_ind in range(num_rounds):# Adjust the LM temperatures of the program predictors- for pred in program.predictors:+ for pred in program.predictors():# If a None temperature is passed, keep the predictor's temperature# as is for the first roundtemp = sampling_temperature_base@@ -335,17 +337,3 @@ class DataCollectionCallback:if self.num_attempts >= self.max_attempts:return []return examples_still_incorrect---callback = DataCollectionCallback(max_attempts=max_attempts)--dc_kwargs = {- "include": None,- "exclude_demos":True,- "temperature": base_temp,- "temperature_delta":0.0001,- "move_on_callback": callback.move_on_callback_correct_with_max,- "num_threads": NUM_THREADS,-}--# results = bootstrap_multiple_prompt_completion_data(program, **kwargs)\ No newline at end of filediff --git a/examples/finetuning/basicmh_3_3_6.json b/examples/finetuning/basicmh_3_3_6.jsonnew file mode 100644index 000000000..37be3f494--- /dev/null+++ b/examples/finetuning/basicmh_3_3_6.json@@ -0,0 +1,340 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [],+ "question": "Singer Annette Snell died in a airplane crash during a forced landing on a highway in what state?",+ "rationale": "determine the location of the airplane crash involving singer Annette Snell. First, we need to identify any news articles, reports, or obituaries that mention the details of Annette Snell's death, including the state where the crash occurred.",+ "search_query": "\"Annette Snell airplane crash state\""+ },+ {+ "question": "Which movie is based on a book, Darby O'Gill and the Little People and College Road Trip?",+ "answer": "Darby O'Gill and the Little People",+ "dspy_uuid": "346d7c7a-9f8d-485b-b9a0-cd3e813e4f79",+ "dspy_split": "train"+ },+ {+ "question": "Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?",+ "answer": "Buena Vista Distribution",+ "dspy_uuid": "1d429a20-dab2-45c8-9c7e-f09c02ff64b7",+ "dspy_split": "train"+ },+ {+ "question": "Which director was nominated for more Academy Awards, Lee Tit or Sidney Lumet?",+ "answer": "14 of his films were nominated",+ "dspy_uuid": "2c8820fb-8a3e-4285-a8f1-f6421ff23bff",+ "dspy_split": "train"+ },+ {+ "question": "A police detective, author and former child actor who appeared on a show that had segments about making food and taking things apart, joined the police department in what year?",+ "answer": "1996",+ "dspy_uuid": "1694f0bc-c3e7-4c3e-8ba6-e7d7afe1c060",+ "dspy_split": "train"+ },+ {+ "question": "Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?",+ "answer": "Rosario Dawson",+ "dspy_uuid": "7aa6f285-aa9f-4302-a740-1ebf9fe25ecd",+ "dspy_split": "train"+ },+ {+ "question": "Who created the TV series which had the storyline Sharongate ?",+ "answer": "Julia Smith and Tony Holland",+ "dspy_uuid": "fa0f165b-2338-4048-b82e-25185e1d5bce",+ "dspy_split": "train"+ },+ {+ "question": "which American actor was Candace Kita guest starred with ",+ "answer": "Bill Murray",+ "dspy_uuid": "6872de0d-e60f-4568-b67b-3c12efcce4df",+ "dspy_split": "train"+ },+ {+ "question": "Remember Me Ballin' is a CD single by Indo G that features an American rapper born in what year?",+ "answer": "1979",+ "dspy_uuid": "c942e7bd-acc4-4f4a-bc52-aec8b6e49567",+ "dspy_split": "train"+ },+ {+ "question": "Both Jack Hill and Bernard McEveety are involved in what occupation?",+ "answer": "film director",+ "dspy_uuid": "0b3e77cf-0913-4a14-a869-80baac72814b",+ "dspy_split": "train"+ },+ {+ "question": "\"The Day I Met Marie\" is a song written by an English songwriter best known as the lead guitarist for what band?",+ "answer": "the Shadows",+ "dspy_uuid": "01cf72ea-4673-4b75-99d1-4ccddd05a74a",+ "dspy_split": "train"+ },+ {+ "question": "Are both Ralph Saenz and Roddy Woomble credited as writers?",+ "answer": "no",+ "dspy_uuid": "9c678b8b-7e9a-4ac2-99fb-705af4bcabd2",+ "dspy_split": "train"+ },+ {+ "question": "The Montenegrin Mountain Hound and Mudhol Hound are both used for what common purpose?",+ "answer": "hunting",+ "dspy_uuid": "b5e89309-95e2-4e15-b1a4-04653e0b79c1",+ "dspy_split": "train"+ },+ {+ "question": "Carlo Ignazio Monza's music formed a ballet based on what 18th century play? ",+ "answer": "Commedia dell'arte",+ "dspy_uuid": "5237b998-157b-4be0-8911-c57aac61113e",+ "dspy_split": "train"+ },+ {+ "question": "Who lived longer, Al-Ghazali or Averroes (Ibn Rushd)?",+ "answer": "Ibn Rushd",+ "dspy_uuid": "5198340d-31cb-468e-b4dd-2a9ffdba341d",+ "dspy_split": "train"+ },+ {+ "question": "Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?",+ "answer": "Tae Kwon Do Times",+ "dspy_uuid": "077a1105-3c21-4d34-8cb7-bab9f6f1076c",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Annette Snell | Annette Snell (March 22, 1945 \u2013 April 4, 1977) was an American rhythm and blues singer who recorded in the 1960s and 1970s. She died in the Southern Airways Flight 242 crash.",+ "Annette Herfkens | Annette Herfkens (Maracaibo, 29 April 1961) is the sole survivor of Vietnam Airlines Flight 474 which crashed into a mountain on November 14, 1992 in Vietnam. She survived eight days with multiple injuries and sustained herself only on rainwater. The plane, a Soviet-made Yakovlev Yak-40 jet built in 1976, was carrying 25 passengers and six crew when it crashed en route to Nha Trang Airport. Some passengers survived the initial impact but died before they could be rescued. Herfkens' fianc\u00e9, who was travelling with her, died instantly upon impact.",+ "1947 Oregon Beechcraft Bonanza crash | On October 28, 1947, Oregon Governor Earl Snell, second in line of succession Oregon State Senate President Marshall Cornett, and Oregon Secretary of State Robert S. Farrell, Jr. were killed in a crash of a Beechcraft Bonanza in stormy weather southwest of Dog Lake in rural Lake County, Oregon. The trio died along with the aircraft's pilot, who was taking them to an isolated area to hunt Canada geese. The tragedy propelled Oregon Speaker of the House John Hubert Hall to the Governorship of Oregon."+ ],+ "question": "Singer Annette Snell died in a airplane crash during a forced landing on a highway in what state?",+ "rationale": "produce the search_query. We need to find information on the circumstances surrounding Annette Snell's death in the airplane crash. Since the context mentions that Annette Snell died in the Southern Airways Flight 242 crash, we should look up details specifically about this crash, including where it occurred. The question asks about the state of the forced landing, so we will search for information regarding the location of this incident.",+ "search_query": "\"Southern Airways Flight 242 crash location state\""+ },+ {+ "question": "Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?",+ "answer": "Rosario Dawson",+ "dspy_uuid": "7aa6f285-aa9f-4302-a740-1ebf9fe25ecd",+ "dspy_split": "train"+ },+ {+ "question": "Which director was nominated for more Academy Awards, Lee Tit or Sidney Lumet?",+ "answer": "14 of his films were nominated",+ "dspy_uuid": "2c8820fb-8a3e-4285-a8f1-f6421ff23bff",+ "dspy_split": "train"+ },+ {+ "question": "Carlo Ignazio Monza's music formed a ballet based on what 18th century play? ",+ "answer": "Commedia dell'arte",+ "dspy_uuid": "5237b998-157b-4be0-8911-c57aac61113e",+ "dspy_split": "train"+ },+ {+ "question": "Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?",+ "answer": "Buena Vista Distribution",+ "dspy_uuid": "1d429a20-dab2-45c8-9c7e-f09c02ff64b7",+ "dspy_split": "train"+ },+ {+ "question": "\"The Day I Met Marie\" is a song written by an English songwriter best known as the lead guitarist for what band?",+ "answer": "the Shadows",+ "dspy_uuid": "01cf72ea-4673-4b75-99d1-4ccddd05a74a",+ "dspy_split": "train"+ },+ {+ "question": "Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?",+ "answer": "Tae Kwon Do Times",+ "dspy_uuid": "077a1105-3c21-4d34-8cb7-bab9f6f1076c",+ "dspy_split": "train"+ },+ {+ "question": "Both Jack Hill and Bernard McEveety are involved in what occupation?",+ "answer": "film director",+ "dspy_uuid": "0b3e77cf-0913-4a14-a869-80baac72814b",+ "dspy_split": "train"+ },+ {+ "question": "Who lived longer, Al-Ghazali or Averroes (Ibn Rushd)?",+ "answer": "Ibn Rushd",+ "dspy_uuid": "5198340d-31cb-468e-b4dd-2a9ffdba341d",+ "dspy_split": "train"+ },+ {+ "question": "Remember Me Ballin' is a CD single by Indo G that features an American rapper born in what year?",+ "answer": "1979",+ "dspy_uuid": "c942e7bd-acc4-4f4a-bc52-aec8b6e49567",+ "dspy_split": "train"+ },+ {+ "question": "Which movie is based on a book, Darby O'Gill and the Little People and College Road Trip?",+ "answer": "Darby O'Gill and the Little People",+ "dspy_uuid": "346d7c7a-9f8d-485b-b9a0-cd3e813e4f79",+ "dspy_split": "train"+ },+ {+ "question": "Who created the TV series which had the storyline Sharongate ?",+ "answer": "Julia Smith and Tony Holland",+ "dspy_uuid": "fa0f165b-2338-4048-b82e-25185e1d5bce",+ "dspy_split": "train"+ },+ {+ "question": "which American actor was Candace Kita guest starred with ",+ "answer": "Bill Murray",+ "dspy_uuid": "6872de0d-e60f-4568-b67b-3c12efcce4df",+ "dspy_split": "train"+ },+ {+ "question": "The Montenegrin Mountain Hound and Mudhol Hound are both used for what common purpose?",+ "answer": "hunting",+ "dspy_uuid": "b5e89309-95e2-4e15-b1a4-04653e0b79c1",+ "dspy_split": "train"+ },+ {+ "question": "Are both Ralph Saenz and Roddy Woomble credited as writers?",+ "answer": "no",+ "dspy_uuid": "9c678b8b-7e9a-4ac2-99fb-705af4bcabd2",+ "dspy_split": "train"+ },+ {+ "question": "A police detective, author and former child actor who appeared on a show that had segments about making food and taking things apart, joined the police department in what year?",+ "answer": "1996",+ "dspy_uuid": "1694f0bc-c3e7-4c3e-8ba6-e7d7afe1c060",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Annette Snell | Annette Snell (March 22, 1945 \u2013 April 4, 1977) was an American rhythm and blues singer who recorded in the 1960s and 1970s. She died in the Southern Airways Flight 242 crash.",+ "Annette Herfkens | Annette Herfkens (Maracaibo, 29 April 1961) is the sole survivor of Vietnam Airlines Flight 474 which crashed into a mountain on November 14, 1992 in Vietnam. She survived eight days with multiple injuries and sustained herself only on rainwater. The plane, a Soviet-made Yakovlev Yak-40 jet built in 1976, was carrying 25 passengers and six crew when it crashed en route to Nha Trang Airport. Some passengers survived the initial impact but died before they could be rescued. Herfkens' fianc\u00e9, who was travelling with her, died instantly upon impact.",+ "1947 Oregon Beechcraft Bonanza crash | On October 28, 1947, Oregon Governor Earl Snell, second in line of succession Oregon State Senate President Marshall Cornett, and Oregon Secretary of State Robert S. Farrell, Jr. were killed in a crash of a Beechcraft Bonanza in stormy weather southwest of Dog Lake in rural Lake County, Oregon. The trio died along with the aircraft's pilot, who was taking them to an isolated area to hunt Canada geese. The tragedy propelled Oregon Speaker of the House John Hubert Hall to the Governorship of Oregon.",+ "Southern Airways Flight 242 | Southern Airways Flight 242 was a DC-9-31 jet, registered N1335U, that executed a forced landing on a highway in New Hope, Paulding County, Georgia, United States after suffering hail damage and losing thrust on both engines in a severe thunderstorm on April 4, 1977.",+ "Southern Airways Flight 932 | Southern Airways Flight 932 was a chartered Southern Airways Douglas DC-9 domestic United States commercial jet flight from Stallings Field (ISO) in Kinston, North Carolina, to Huntington Tri-State Airport\/Milton J. Ferguson Field (HTS) in Ceredo, West Virginia. At 7:36 pm on November 14, 1970, the aircraft crashed into a hill just short of the Tri-State Airport, killing all 75 people on board.",+ "Southern Airways Flight 49 | The hijacking of Southern Airways Flight 49 started on November 10, 1972 in Birmingham, Alabama, stretching over 30 hours and 4000 mi , not ending until the next evening in Havana, Cuba. Melvin Cale, Louis Moore, and Henry D. Jackson, Jr. successfully hijacked a Southern Airways Douglas DC-9 that was scheduled to fly from Memphis, Tennessee to Miami, Florida via Birmingham and Montgomery, Alabama and Orlando, Florida. The three were each facing criminal charges for unrelated incidents. 34 people, including 31 passengers and 3 crew members, were aboard the airplane when it was hijacked. The hijackers' threat to crash the aircraft into a nuclear reactor led directly to the requirement that U.S. airline passengers be physically screened, beginning January 5, 1973."+ ],+ "question": "Singer Annette Snell died in a airplane crash during a forced landing on a highway in what state?",+ "rationale": "determine the state where the forced landing occurred. The context mentions Southern Airways Flight 242, which executed a forced landing in New Hope, Paulding County, Georgia, United States. Since Annette Snell died in this crash and it was in Georgia, we can conclude that the state is Georgia.",+ "answer": "Georgia"+ },+ {+ "question": "Both Jack Hill and Bernard McEveety are involved in what occupation?",+ "answer": "film director",+ "dspy_uuid": "0b3e77cf-0913-4a14-a869-80baac72814b",+ "dspy_split": "train"+ },+ {+ "question": "Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?",+ "answer": "Tae Kwon Do Times",+ "dspy_uuid": "077a1105-3c21-4d34-8cb7-bab9f6f1076c",+ "dspy_split": "train"+ },+ {+ "question": "Which movie is based on a book, Darby O'Gill and the Little People and College Road Trip?",+ "answer": "Darby O'Gill and the Little People",+ "dspy_uuid": "346d7c7a-9f8d-485b-b9a0-cd3e813e4f79",+ "dspy_split": "train"+ },+ {+ "question": "Who created the TV series which had the storyline Sharongate ?",+ "answer": "Julia Smith and Tony Holland",+ "dspy_uuid": "fa0f165b-2338-4048-b82e-25185e1d5bce",+ "dspy_split": "train"+ },+ {+ "question": "Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?",+ "answer": "Buena Vista Distribution",+ "dspy_uuid": "1d429a20-dab2-45c8-9c7e-f09c02ff64b7",+ "dspy_split": "train"+ },+ {+ "question": "Remember Me Ballin' is a CD single by Indo G that features an American rapper born in what year?",+ "answer": "1979",+ "dspy_uuid": "c942e7bd-acc4-4f4a-bc52-aec8b6e49567",+ "dspy_split": "train"+ },+ {+ "question": "Who lived longer, Al-Ghazali or Averroes (Ibn Rushd)?",+ "answer": "Ibn Rushd",+ "dspy_uuid": "5198340d-31cb-468e-b4dd-2a9ffdba341d",+ "dspy_split": "train"+ },+ {+ "question": "The Montenegrin Mountain Hound and Mudhol Hound are both used for what common purpose?",+ "answer": "hunting",+ "dspy_uuid": "b5e89309-95e2-4e15-b1a4-04653e0b79c1",+ "dspy_split": "train"+ },+ {+ "question": "A police detective, author and former child actor who appeared on a show that had segments about making food and taking things apart, joined the police department in what year?",+ "answer": "1996",+ "dspy_uuid": "1694f0bc-c3e7-4c3e-8ba6-e7d7afe1c060",+ "dspy_split": "train"+ },+ {+ "question": "\"The Day I Met Marie\" is a song written by an English songwriter best known as the lead guitarist for what band?",+ "answer": "the Shadows",+ "dspy_uuid": "01cf72ea-4673-4b75-99d1-4ccddd05a74a",+ "dspy_split": "train"+ },+ {+ "question": "Carlo Ignazio Monza's music formed a ballet based on what 18th century play? ",+ "answer": "Commedia dell'arte",+ "dspy_uuid": "5237b998-157b-4be0-8911-c57aac61113e",+ "dspy_split": "train"+ },+ {+ "question": "Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?",+ "answer": "Rosario Dawson",+ "dspy_uuid": "7aa6f285-aa9f-4302-a740-1ebf9fe25ecd",+ "dspy_split": "train"+ },+ {+ "question": "Are both Ralph Saenz and Roddy Woomble credited as writers?",+ "answer": "no",+ "dspy_uuid": "9c678b8b-7e9a-4ac2-99fb-705af4bcabd2",+ "dspy_split": "train"+ },+ {+ "question": "which American actor was Candace Kita guest starred with ",+ "answer": "Bill Murray",+ "dspy_uuid": "6872de0d-e60f-4568-b67b-3c12efcce4df",+ "dspy_split": "train"+ },+ {+ "question": "Which director was nominated for more Academy Awards, Lee Tit or Sidney Lumet?",+ "answer": "14 of his films were nominated",+ "dspy_uuid": "2c8820fb-8a3e-4285-a8f1-f6421ff23bff",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynbindex 679ddd84d..e56e93f51 100644--- a/examples/finetuning/openai_finetuning_hpqa.ipynb+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb@@ -2,9 +2,18 @@"cells": [{"cell_type": "code",- "execution_count": 1,+ "execution_count": 5,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "The autoreload extension is already loaded. To reload it, use:\n",+ " %reload_ext autoreload\n"+ ]+ }+ ],"source": ["%load_ext autoreload\n","%autoreload 2"@@ -12,20 +21,9 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": 6,"metadata": {},- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",- " warnings.warn(\n",- "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",- " from .autonotebook import tqdm as notebook_tqdm\n"- ]- }- ],+ "outputs": [],"source": ["import sys\n","import os\n",@@ -49,7 +47,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 7,"metadata": {},"outputs": [],"source": [@@ -58,19 +56,19 @@"openai.api_key = os.environ.get('OPENAI_API_KEY')\n","\n","mini = \"gpt-4o-mini-2024-07-18\"\n",- "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n","base_temp = 0.9\n","\n",- "prompt_model = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",+ "lm = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n","\n",+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n","colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n","\n",- "dspy.settings.configure(rm=colbertv2) # "+ "dspy.settings.configure(rm=colbertv2, lm=lm) "]},{"cell_type": "code",- "execution_count": 4,+ "execution_count": 8,"metadata": {},"outputs": [],"source": [@@ -97,16 +95,16 @@" \n","evaluator = AnswerCorrectness()\n","\n",- "def metric(example, pred, trace=None):\n",- " result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",- " return \"true\" in result and not \"false\" in result\n",+ "# def metric(example, pred, trace=None):\n",+ "# result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",+ "# return \"true\" in result and not \"false\" in result\n","\n","metric = dspy.evaluate.answer_exact_match"]},{"cell_type": "code",- "execution_count": 5,+ "execution_count": 9,"metadata": {},"outputs": [],"source": [@@ -124,7 +122,7 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": 10,"metadata": {},"outputs": [],"source": [@@ -134,7 +132,7 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": 11,"metadata": {},"outputs": [],"source": [@@ -161,7 +159,17 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": 12,+ "metadata": {},+ "outputs": [],+ "source": [+ "# program = BasicMH()\n",+ "# program(question=\"What is the capital of France?\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,"metadata": {},"outputs": [],"source": [@@ -174,15 +182,14 @@"COMPILE = False\n","\n","if COMPILE:\n",- " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",- " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",- " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",- " teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",- " basicmh_bs = teleprompter.compile(BasicMH(**program_params), trainset=trainset[:100], valset=devset[:250])\n",- " basicmh_bs.save(f\"basicmh_{max_bootstrapped_demos}_{max_labeled_demos}_{num_candidate_programs}.json\")\n",- "\n",- " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:300])\n",- " bs_eval = evaluate(basicmh_bs, devset=devset[:300])\n",+ " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",+ " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",+ " teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",+ " basicmh_bs = teleprompter.compile(BasicMH(**program_params), trainset=trainset[:100], valset=devset[:150])\n",+ " basicmh_bs.save(f\"basicmh_{max_bootstrapped_demos}_{max_labeled_demos}_{num_candidate_programs}.json\")\n",+ "\n",+ " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:300])\n",+ " bs_eval = evaluate(basicmh_bs, devset=devset[:300])\n","else:\n"," basicmh_bs = BasicMH(**program_params)\n"," basicmh_bs.load(\"basicmh_3_3_6.json\")"@@ -190,62 +197,57 @@},{"cell_type": "code",- "execution_count": 9,+ "execution_count": 14,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 355.76it/s]\n"+ ]+ }+ ],"source": ["from collections import Counter\n",- "from dspy.teleprompt.finetune_teleprompter import bootstrap_multiple_prompt_completion_data\n",+ "from dspy.teleprompt.finetune_teleprompter import bootstrap_data_multiple_rounds, DataCollectionCallback\n","from typing import Callable\n","\n",- "samples = 500\n",- "max_attempts = 1\n",+ "samples = 100\n","\n","\n",- "class DataCollectionCallback:\n",- " def __init__(self):\n",- " self.num_attempts = 0\n",- " self.num_correct = 2\n",- " self.max_attempts = max_attempts\n",- "\n",- " def move_on_callback_correct_with_max(self, dataset_copy, data):\n",- " correct_counts = Counter(list(map(lambda x: x[\"example\"], data)))\n",- " examples_still_incorrect = [x for x in dataset_copy if correct_counts.get(x, 0) < self.num_correct]\n",- " self.num_attempts += 1\n",- " if self.num_attempts >= self.max_attempts:\n",- " return []\n",- " return examples_still_incorrect\n",- "\n",- "\n",- "callback = DataCollectionCallback()\n",- " # \"dataset\":devset[:samples],\n",- " # \"metric\":metric,\n",+ "\n",+ "callback = DataCollectionCallback(num_correct=1, max_attempts=1)\n",+ "\n",+ "dspy.settings.configure(experimental=True)\n","dc_kwargs = {\n",- " \"include\": None,\n",- " \"exclude_demos\":True, \n",- " \"temperature\": base_temp,\n",- " \"temperature_delta\":0.0001,\n",- " \"move_on_callback\": callback.move_on_callback_correct_with_max,\n",+ " # \"exclude_demos\":True, \n",+ " \"sampling_temperature_base\": base_temp,\n",+ " \"sampling_temperature_delta\":0.0001,\n",+ " \"next_round_dataset_callback\": callback.move_on_callback_correct_with_max,\n"," \"num_threads\": NUM_THREADS,\n","}\n",+ "basicmh_bs = BasicMH(**program_params)\n",+ "basicmh_bs.load(\"basicmh_3_3_6.json\")\n","\n",- "# results = bootstrap_multiple_prompt_completion_data(program, **kwargs)"+ "results = bootstrap_data_multiple_rounds(basicmh_bs, devset[:samples], metric, num_rounds=1, **dc_kwargs)\n",+ "# results[:2]"]},{"cell_type": "code",- "execution_count": 10,+ "execution_count": 15,"metadata": {},"outputs": [],"source": [- "from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",- "\n",- "teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"+ "# from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",+ "# ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\n",+ "# teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"]},{"cell_type": "code",- "execution_count": 11,+ "execution_count": 16,"metadata": {},"outputs": [],"source": [@@ -253,374 +255,45 @@"\n","if TRAIN_FROM_SCRATCH:\n"," teacher = basicmh_bs\n",- " lm = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",- " # for predictor in teacher.predictors():\n",- " # predictor.lm = lm\n",- " dspy.settings.configure(lm=lm)\n",+ " \n",+ " # something like\n",+ " # Go through all the possible methods + save/load\n",+ " # change to future[lm] first\n",+ "\n",+ "\n"," # TODO: Working here\n"," # teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)\n",- " kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",- " # NOTE: LM IS NOT SET PROPERLY\n",- " basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",- " basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",- "else:\n",- " basicmh_bs_ft = BasicMH(**program_params)\n",- " basicmh_bs_ft.load('mini_ft_hpqa_100.json')"+ "# kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",+ "# # NOTE: LM IS NOT SET PROPERLY\n",+ "# basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",+ "# basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",+ "# else:\n",+ "# basicmh_bs_ft = BasicMH(**program_params)\n",+ "# basicmh_bs_ft.load('mini_ft_hpqa_100.json')"]},{"cell_type": "code",- "execution_count": 12,+ "execution_count": 17,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "ft:gpt-4o-mini-2024-07-18:anyscale::9tkXPhLI\n"- ]- }- ],+ "outputs": [],"source": [- "print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",- "dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"+ "lm2 = lm.copy()\n",+ "lm2.kwargs[\"model\"] = \"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\"\n",+ "\n",+ "basicmh_bs_ft = BasicMH(**program_params)\n",+ "basicmh_bs_ft._set_all_predictor_lms(lm2)\n",+ "# print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",+ "# dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"]},{"cell_type": "code",- "execution_count": 13,+ "execution_count": 18,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Going to sample between 1 and 3 traces per predictor.\n",- "Will attempt to bootstrap 6 candidate sets.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 9 / 10 (90.0): 4%|▎ | 9/250 [00:00<00:01, 136.55it/s]"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 188 / 250 (75.2): 100%|██████████| 250/250 [00:00<00:00, 257.17it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 75.2 for set: [0, 0, 0]\n",- "New best sscore: 75.2 for seed -3\n",- "Scores so far: [75.2]\n",- "Best score: 75.2\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 183 / 250 (73.2): 100%|██████████| 250/250 [00:00<00:00, 533.70it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 73.2 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2]\n",- "Best score: 75.2\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 4%|▍ | 4/100 [00:00<00:00, 173.77it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 3 full traces after 5 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 184 / 250 (73.6): 100%|██████████| 250/250 [00:00<00:00, 296.67it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 73.6 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.784\n",- "Average of max per entry across top 3 scores: 0.8\n",- "Average of max per entry across top 5 scores: 0.8\n",- "Average of max per entry across top 8 scores: 0.8\n",- "Average of max per entry across top 9999 scores: 0.8\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 3%|▎ | 3/100 [00:00<00:00, 240.60it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 2 full traces after 4 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 260.19it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 72.8 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.784\n",- "Average of max per entry across top 3 scores: 0.8\n",- "Average of max per entry across top 5 scores: 0.812\n",- "Average of max per entry across top 8 scores: 0.812\n",- "Average of max per entry across top 9999 scores: 0.812\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 1/100 [00:00<00:00, 154.99it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 2 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 178 / 250 (71.2): 100%|██████████| 250/250 [00:00<00:00, 310.60it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 71.2 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.784\n",- "Average of max per entry across top 3 scores: 0.8\n",- "Average of max per entry across top 5 scores: 0.824\n",- "Average of max per entry across top 8 scores: 0.824\n",- "Average of max per entry across top 9999 scores: 0.824\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 1/100 [00:00<00:00, 451.92it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 2 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 305.13it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 72.8 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.784\n",- "Average of max per entry across top 3 scores: 0.8\n",- "Average of max per entry across top 5 scores: 0.824\n",- "Average of max per entry across top 8 scores: 0.832\n",- "Average of max per entry across top 9999 scores: 0.832\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 2%|▏ | 2/100 [00:00<00:00, 148.98it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 3 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:00<00:00, 289.52it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 74.4 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.796\n",- "Average of max per entry across top 3 scores: 0.812\n",- "Average of max per entry across top 5 scores: 0.828\n",- "Average of max per entry across top 8 scores: 0.84\n",- "Average of max per entry across top 9999 scores: 0.84\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 1/100 [00:00<00:00, 258.21it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 2 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 185 / 250 (74.0): 100%|██████████| 250/250 [00:00<00:00, 346.57it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 74.0 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.796\n",- "Average of max per entry across top 3 scores: 0.816\n",- "Average of max per entry across top 5 scores: 0.836\n",- "Average of max per entry across top 8 scores: 0.848\n",- "Average of max per entry across top 9999 scores: 0.848\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 3%|▎ | 3/100 [00:00<00:00, 258.47it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 3 full traces after 4 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:02<00:00, 113.88it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Score: 74.4 for set: [16, 16, 16]\n",- "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0, 74.4]\n",- "Best score: 75.2\n",- "Average of max per entry across top 1 scores: 0.752\n",- "Average of max per entry across top 2 scores: 0.796\n",- "Average of max per entry across top 3 scores: 0.812\n",- "Average of max per entry across top 5 scores: 0.832\n",- "Average of max per entry across top 8 scores: 0.844\n",- "Average of max per entry across top 9999 scores: 0.848\n",- "9 candidate programs found.\n",- "[('retrieve', <dspy.retrieve.retrieve.Retrieve object at 0x124d86610>), ('generate_query[0]', Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- "))), ('generate_query[1]', Predict(StringSignature(context, question -> rationale, search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- "))), ('generate_answer', Predict(StringSignature(context, question -> rationale, answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the answer}. We ...', '__dspy_field_type': 'output'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- ")))]\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"- ]- }- ],+ "outputs": [],"source": [- "RECOMPILE_FT_MODEL = True\n",+ "RECOMPILE_FT_MODEL = False\n","\n","if RECOMPILE_FT_MODEL:\n"," max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",@@ -635,47 +308,45 @@},{"cell_type": "code",- "execution_count": 16,+ "execution_count": 19,"metadata": {},"outputs": [{"name": "stderr","output_type": "stream","text": [- "Average Metric: 219 / 300 (73.0): 100%|██████████| 300/300 [00:00<00:00, 4214.14it/s]\n",- "Average Metric: 227 / 300 (75.7): 100%|██████████| 300/300 [00:00<00:00, 2704.22it/s]\n",- "Average Metric: 216 / 300 (72.0): 100%|██████████| 300/300 [00:00<00:00, 2398.68it/s]\n",- "Average Metric: 222 / 300 (74.0): 100%|██████████| 300/300 [00:00<00:00, 2448.68it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N=500 and up to 1 attempts for each example with one model for all predictors. Tested on first 300 of devset.\n",- "Non-finetuned model: 73.0\n",- "Non-finetuned bootstrapped model: 75.67\n",- "Finetuned model: 72.0\n",- "Finetuned model with bootstrapping: 74.0\n"+ "Average Metric: 33 / 69 (47.8): 23%|██▎ | 69/300 [00:25<01:03, 3.64it/s]WARNING:dspy.evaluate.evaluate:\u001b[2m2024-08-13T00:56:16.101839Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mReceived SIGINT. Cancelling evaluation.\u001b[0m [\u001b[0m\u001b[1m\u001b[34mdspy.evaluate.evaluate\u001b[0m]\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mevaluate.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m91\u001b[0m\n"]},{- "name": "stderr",- "output_type": "stream",- "text": [- "\n"+ "ename": "KeyboardInterrupt",+ "evalue": "",+ "output_type": "error",+ "traceback": [+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",+ "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",+ "Cell \u001b[0;32mIn[19], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m TEST_SIZE \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m dspy\u001b[38;5;241m.\u001b[39mcontext(lm\u001b[38;5;241m=\u001b[39mdspy\u001b[38;5;241m.\u001b[39mOpenAI(model\u001b[38;5;241m=\u001b[39mmini, max_tokens\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m500\u001b[39m, temperature\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.0\u001b[39m)):\n\u001b[0;32m----> 3\u001b[0m baseline_eval \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBasicMH\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mprogram_params\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevset\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43mTEST_SIZE\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m bs_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[1;32m 5\u001b[0m bs_ft_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs_ft, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n",+ "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:193\u001b[0m, in \u001b[0;36mEvaluate.__call__\u001b[0;34m(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)\u001b[0m\n\u001b[1;32m 191\u001b[0m reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute_single_thread(wrapped_program, devset, display_progress)\n\u001b[1;32m 192\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 193\u001b[0m reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_multi_thread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[43m \u001b[49m\u001b[43mwrapped_program\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 195\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 196\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_threads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43mdisplay_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 200\u001b[0m dspy\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAverage Metric: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mncorrect\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m / \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mntotal\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mround\u001b[39m(\u001b[38;5;241m100\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39mncorrect\u001b[38;5;250m \u001b[39m\u001b[38;5;241m/\u001b[39m\u001b[38;5;250m \u001b[39mntotal,\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m%)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 202\u001b[0m predicted_devset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m(reordered_devset)\n",+ "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:109\u001b[0m, in \u001b[0;36mEvaluate._execute_multi_thread\u001b[0;34m(self, wrapped_program, devset, num_threads, display_progress)\u001b[0m\n\u001b[1;32m 106\u001b[0m futures \u001b[38;5;241m=\u001b[39m {executor\u001b[38;5;241m.\u001b[39msubmit(cancellable_wrapped_program, idx, arg) \u001b[38;5;28;01mfor\u001b[39;00m idx, arg \u001b[38;5;129;01min\u001b[39;00m devset}\n\u001b[1;32m 107\u001b[0m pbar \u001b[38;5;241m=\u001b[39m tqdm\u001b[38;5;241m.\u001b[39mtqdm(total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(devset), dynamic_ncols\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, disable\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m display_progress)\n\u001b[0;32m--> 109\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m as_completed(futures):\n\u001b[1;32m 110\u001b[0m example_idx, example, prediction, score \u001b[38;5;241m=\u001b[39m future\u001b[38;5;241m.\u001b[39mresult()\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# use the cancelled_job literal to check if the job was cancelled - use \"is\" not \"==\"\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;66;03m# in case the prediction is \"cancelled\" for some reason.\u001b[39;00m\n",+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:245\u001b[0m, in \u001b[0;36mas_completed\u001b[0;34m(fs, timeout)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wait_timeout \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 241\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\n\u001b[1;32m 242\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m (of \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m) futures unfinished\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28mlen\u001b[39m(pending), total_futures))\n\u001b[0;32m--> 245\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwait_timeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m waiter\u001b[38;5;241m.\u001b[39mlock:\n\u001b[1;32m 248\u001b[0m finished \u001b[38;5;241m=\u001b[39m waiter\u001b[38;5;241m.\u001b[39mfinished_futures\n",+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py:574\u001b[0m, in \u001b[0;36mEvent.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 572\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flag\n\u001b[1;32m 573\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m signaled:\n\u001b[0;32m--> 574\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 575\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m signaled\n",+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py:312\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 313\u001b[0m gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",+ "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dspy/evaluate/evaluate.py:92\u001b[0m, in \u001b[0;36mEvaluate._execute_multi_thread.<locals>.interrupt_handler_manager.<locals>.interrupt_handler\u001b[0;34m(sig, frame)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel_jobs\u001b[38;5;241m.\u001b[39mset()\n\u001b[1;32m 91\u001b[0m dspy\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReceived SIGINT. Cancelling evaluation.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 92\u001b[0m \u001b[43mdefault_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[43msig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mframe\u001b[49m\u001b[43m)\u001b[49m\n",+ "\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}],"source": ["TEST_SIZE = 300\n",+ "\n","with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n"," baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n"," bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",- "bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",- "bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",+ "with dspy.context(lm=dspy.OpenAI(model=\"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\", max_tokens=500, temperature=0.0)):\n",+ " bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",+ " bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n","\n",- "print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to {max_attempts} attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n",+ "print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to 1 attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n","print(f\"Non-finetuned model: {baseline_eval}\")\n","print(f\"Non-finetuned bootstrapped model: {bs_eval}\")\n","print(f\"Finetuned model: {bs_ft_eval}\")\n",