DSPy: The framework for programming—not prompting—language models
Python
Review changes
Isaac Millerb1ddecc2b08e
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 4b3fbec2d..42cf8222f 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -21,7 +21,7 @@ from .hf_client import Anyscale, HFClientTGI, Togetherfrom .llama import *from .mistral import *from .ollama import *-from .openai import *+from .trainable_openai import *from .multi_openai import MultiOpenAIfrom .premai import PremAIfrom .pyserini import *diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex a06e23236..9509288ea 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -2,6 +2,7 @@ from abc import ABC, abstractmethodfrom asyncio import Futurefrom concurrent.futures import ThreadPoolExecutorfrom copy import deepcopy+from enum import Enumfrom typing import Literal, Optionalclass LM(ABC):@@ -144,10 +145,13 @@ class LM(ABC):return self.__class__(model=model, **kwargs)-FinetuningMethod = Literal["SFT", "Contrastive"]-class FinetunableLM(LM, ABC):- def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLM']:- future: Future['FinetunableLM'] = Future()+class TrainingMethod(Enum):+ SFT = "SFT"+ Contrastive = "Contrastive"++class TrainableLM(LM, ABC):+ def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['TrainableLM']:+ future: Future['TrainableLM'] = Future()new_lm = deepcopy(self)@@ -159,7 +163,7 @@ class FinetunableLM(LM, ABC):return future@abstractmethod- def start_training(self, future: Future['FinetunableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):+ def start_training(self, future: Future['TrainableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):pass@abstractmethoddiff --git a/dsp/modules/openai.py b/dsp/modules/trainable_openai.pysimilarity index 90%rename from dsp/modules/openai.pyrename to dsp/modules/trainable_openai.pyindex 58abb381d..8535ff2fa 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/trainable_openai.py@@ -3,7 +3,7 @@ import timefrom typing import Any, List, Optional, Literal, Unionimport ujsonimport openai-from dsp.modules.lm import FinetunableLM, FinetuningMethod+from dsp.modules.lm import TrainableLM, FinetuningMethodfrom dsp.modules.gpt3 import GPT3from collections import defaultdict@@ -151,7 +151,7 @@ def backoff_hdlr(details):"{kwargs}".format(**details),)-class OpenAIModel(GPT3, FinetunableLM):+class TrainableOpenAI(GPT3, TrainableLM):"""Wrapper around specifically the OpenAI API to finetune.Args:@@ -160,8 +160,6 @@ class OpenAIModel(GPT3, FinetunableLM):api_provider (Literal["openai"], optional): The API provider to use. Defaults to "openai".model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".system_prompt (Optional[str], optional): The system prompt to use. Defaults to None in init, and "You are a helpful assistant." in format_data_for_vanilla_finetuning.- fine_tuning_file_ids (dict, optional): The file ids of the training and validation data. Defaults to {}.- fine_tuning_job_id (Optional[str], optional): The job id of the fine-tuning job. Defaults to None. Again, checkpointing is not supported ATM but this would be the place to store it.**kwargs: Additional arguments to pass to the API provider."""@@ -173,16 +171,12 @@ class OpenAIModel(GPT3, FinetunableLM):api_base: Optional[str] = None,model_type: Literal["chat", "text"] = None,system_prompt: Optional[str] = None,- fine_tuning_file_ids: dict = {},- fine_tuning_job_id: Optional[str] = None,**kwargs,):super().__init__(model, api_key=api_key, api_provider=api_provider, api_base=api_base, model_type=model_type, system_prompt=system_prompt, **kwargs)assert self.provider == "openai", "You must use an OpenAI model with this class."- self.fine_tuning_file_ids = fine_tuning_file_ids- self.fine_tuning_job_id = fine_tuning_job_id- def verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:+ def _verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:"""Verify the training arguments before starting training.More information on dataset verification can be found here: https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset@@ -222,7 +216,7 @@ class OpenAIModel(GPT3, FinetunableLM):return True- def format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, Any]]:+ def _format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, Any]]:def format_single_item(item):messages = [{"role": "user", "content": item["prompt"]}, {"role": "assistant", "content": item["completion"]}]@@ -255,7 +249,7 @@ class OpenAIModel(GPT3, FinetunableLM):self.fine_tuning_file_ids[name] = file.id# TODO: support OAI wandb integration- def start_remote_training(self, **kwargs) -> str:+ def _start_remote_training(self, **kwargs) -> str:assert self.fine_tuning_file_ids["train"] is not None, "You must load data before starting training"hyperparameters = kwargs.get("hyperparameters", {})job = openai.fine_tuning.jobs.create(@@ -330,7 +324,7 @@ class OpenAIModel(GPT3, FinetunableLM):else:raise RuntimeError("Job not completed yet, cannot retrieve model")- def start_training(self, future: Future['OpenAIModel'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):+ def start_training(self, future: Future['TrainableOpenAI'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):"""Handles the fine-tuning process for an OpenAIModel instance.@@ -345,15 +339,14 @@ class OpenAIModel(GPT3, FinetunableLM):traindataset = ujson.load(open(train_path))valdataset = ujson.load(open(val_path)) if val_path else None- self.verify_training_arguments(traindataset, valdataset, kwargs)+ self._verify_training_arguments(traindataset, valdataset, kwargs)self._load_data(train_path, val_path)# Start the remote training- job_id = self.start_remote_training(**kwargs)+ job_id = self._start_remote_training(**kwargs)# Wait for the training to complete- while not self.check_training_status():- time.sleep(60) # Check every 60 seconds+ self.wait_for_training()# Retrieve the trained model and return a copyself.retrieve_trained_model_client()@@ -363,7 +356,6 @@ class OpenAIModel(GPT3, FinetunableLM):future.set_exception(e)def wait_for_training(self):- # Wait for the future to completewhile not self.check_training_status():time.sleep(60)@@ -378,6 +370,6 @@ class OpenAIModel(GPT3, FinetunableLM):job = openai.fine_tuning.jobs.retrieve(job_id)if job.status != "succeeded":raise RuntimeError("Job not completed yet, cannot retrieve model")- model = OpenAIModel(model=job.fine_tuned_model, **kwargs)+ model = TrainableOpenAI(model=job.fine_tuned_model, **kwargs)model.fine_tuning_job_id = job_idreturn model\ No newline at end of filediff --git a/dspy/__init__.py b/dspy/__init__.pyindex 7cba0e252..05d2a6d48 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -39,7 +39,7 @@ Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal-OpenAIModel = dsp.OpenAIModel+TrainableOpenAI = dsp.TrainableOpenAILlamaCpp = dsp.LlamaCppBedrock = dsp.Bedrockdiff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynbindex cab90ad4c..bd9643bd7 100644--- a/examples/finetuning/openai_finetuning_hpqa.ipynb+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb@@ -171,21 +171,21 @@"name": "stderr","output_type": "stream","text": [- "Average Metric: 257 / 500 (51.4): 100%|██████████| 500/500 [00:01<00:00, 342.65it/s]\n"+ "Average Metric: 259 / 500 (51.8): 100%|██████████| 500/500 [00:01<00:00, 321.91it/s]\n"]},{"name": "stdout","output_type": "stream","text": [- "Writing dataset with length 771 to trainset_data.jsonl\n"+ "Writing dataset with length 777 to trainset_data.jsonl\n"]},{"name": "stderr","output_type": "stream","text": [- "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:43<00:00, 2.87it/s]"+ "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:00<00:00, 350.95it/s]\n"]},{@@ -194,13 +194,6 @@"text": ["Writing dataset with length 228 to devset_data.jsonl\n"]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"- ]}],"source": [@@ -242,34 +235,21 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": 10,"metadata": {},- "outputs": [- {- "ename": "AttributeError",- "evalue": "'_asyncio.Future' object has no attribute '_condition'",- "output_type": "error",- "traceback": [- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",- "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m future_lm \u001b[38;5;241m=\u001b[39m lm\u001b[38;5;241m.\u001b[39mget_finetune(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrainset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdevset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSFT\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# finetuned_lm = future_lm.result()\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m finetuned_lm \u001b[38;5;241m=\u001b[39m \u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfuture_lm\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_when\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIRST_COMPLETED\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mresult() \n\u001b[1;32m 6\u001b[0m finetuned_lm\u001b[38;5;241m.\u001b[39mkwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n",- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:289\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(fs, timeout, return_when)\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait\u001b[39m(fs, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, return_when\u001b[38;5;241m=\u001b[39mALL_COMPLETED):\n\u001b[1;32m 266\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Wait for the futures in the given sequence to complete.\u001b[39;00m\n\u001b[1;32m 267\u001b[0m \n\u001b[1;32m 268\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;124;03m futures.\u001b[39;00m\n\u001b[1;32m 288\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _AcquireFutures(fs):\n\u001b[1;32m 290\u001b[0m done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m fs\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED_AND_NOTIFIED, FINISHED])\n\u001b[1;32m 292\u001b[0m not_done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(fs) \u001b[38;5;241m-\u001b[39m done\n",- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:151\u001b[0m, in \u001b[0;36m_AcquireFutures.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__enter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 150\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfutures:\n\u001b[0;32m--> 151\u001b[0m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241m.\u001b[39macquire()\n",- "\u001b[0;31mAttributeError\u001b[0m: '_asyncio.Future' object has no attribute '_condition'"- ]- }- ],+ "outputs": [],"source": ["import concurrent.futures\n",- "\n",- "future_lm = lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",- "finetuned_lm = await future_lm\n",+ "# lm.()\n",+ "# lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",+ "future_lm = dspy.OpenAIModel.load_from_job_id(\"ftjob-tmccSJv170p4gNFJUYwXdCSq\")\n",+ "finetuned_lm = future_lm\n","finetuned_lm.kwargs[\"temperature\"] = 0.0"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 11,"metadata": {},"outputs": [],"source": [@@ -278,7 +258,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 12,"metadata": {},"outputs": [],"source": [@@ -288,9 +268,361 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 15,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Going to sample between 1 and 3 traces per predictor.\n",+ "Will attempt to bootstrap 6 candidate sets.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 106 / 250 (42.4): 100%|██████████| 250/250 [00:03<00:00, 71.56it/s] \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 42.4 for set: [0, 0, 0]\n",+ "New best score: 42.4 for seed -3\n",+ "Scores so far: [42.4]\n",+ "Best score: 42.4\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 134 / 250 (53.6): 100%|██████████| 250/250 [00:29<00:00, 8.54it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 53.6 for set: [16, 16, 16]\n",+ "New best score: 53.6 for seed -2\n",+ "Scores so far: [42.4, 53.6]\n",+ "Best score: 53.6\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 6%|▌ | 6/100 [00:26<07:02, 4.50s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 7 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 89 / 162 (54.9): 64%|██████▍ | 161/250 [01:00<00:34, 2.62it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 4 / 8 (50.0): 3%|▎ | 8/250 [10:14<5:10:00, 76.86s/it]"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 90 / 163 (55.2): 65%|██████▌ | 163/250 [01:00<00:26, 3.30it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 142 / 250 (56.8): 100%|██████████| 250/250 [01:31<00:00, 2.74it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 56.8 for set: [16, 16, 16]\n",+ "New best score: 56.8 for seed -1\n",+ "Scores so far: [42.4, 53.6, 56.8]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.64\n",+ "Average of max per entry across top 3 scores: 0.656\n",+ "Average of max per entry across top 5 scores: 0.656\n",+ "Average of max per entry across top 8 scores: 0.656\n",+ "Average of max per entry across top 9999 scores: 0.656\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 3%|▎ | 3/100 [00:15<08:17, 5.13s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 4 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 133 / 250 (53.2): 100%|██████████| 250/250 [01:49<00:00, 2.28it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 53.2 for set: [16, 16, 16]\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.64\n",+ "Average of max per entry across top 3 scores: 0.664\n",+ "Average of max per entry across top 5 scores: 0.676\n",+ "Average of max per entry across top 8 scores: 0.676\n",+ "Average of max per entry across top 9999 scores: 0.676\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:04<07:14, 4.39s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 136 / 250 (54.4): 100%|██████████| 250/250 [01:49<00:00, 2.29it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 54.4 for set: [16, 16, 16]\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.624\n",+ "Average of max per entry across top 3 scores: 0.664\n",+ "Average of max per entry across top 5 scores: 0.684\n",+ "Average of max per entry across top 8 scores: 0.684\n",+ "Average of max per entry across top 9999 scores: 0.684\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 2%|▏ | 2/100 [00:06<05:32, 3.39s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 135 / 250 (54.0): 100%|██████████| 250/250 [01:46<00:00, 2.34it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 54.0 for set: [16, 16, 16]\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.624\n",+ "Average of max per entry across top 3 scores: 0.648\n",+ "Average of max per entry across top 5 scores: 0.68\n",+ "Average of max per entry across top 8 scores: 0.688\n",+ "Average of max per entry across top 9999 scores: 0.688\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 2%|▏ | 2/100 [00:10<08:25, 5.16s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 125 / 250 (50.0): 100%|██████████| 250/250 [13:00<00:00, 3.12s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 50.0 for set: [16, 16, 16]\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.624\n",+ "Average of max per entry across top 3 scores: 0.648\n",+ "Average of max per entry across top 5 scores: 0.68\n",+ "Average of max per entry across top 8 scores: 0.688\n",+ "Average of max per entry across top 9999 scores: 0.688\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [17:22<28:40:26, 1042.69s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 133 / 250 (53.2): 100%|██████████| 250/250 [01:35<00:00, 2.61it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 53.2 for set: [16, 16, 16]\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0, 53.2]\n",+ "Best score: 56.8\n",+ "Average of max per entry across top 1 scores: 0.568\n",+ "Average of max per entry across top 2 scores: 0.624\n",+ "Average of max per entry across top 3 scores: 0.648\n",+ "Average of max per entry across top 5 scores: 0.68\n",+ "Average of max per entry across top 8 scores: 0.688\n",+ "Average of max per entry across top 9999 scores: 0.688\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 8%|▊ | 8/100 [00:44<08:35, 5.60s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 9 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 143 / 250 (57.2): 100%|██████████| 250/250 [01:47<00:00, 2.33it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 57.2 for set: [16, 16, 16]\n",+ "New best score: 57.2 for seed 5\n",+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0, 53.2, 57.2]\n",+ "Best score: 57.2\n",+ "Average of max per entry across top 1 scores: 0.572\n",+ "Average of max per entry across top 2 scores: 0.64\n",+ "Average of max per entry across top 3 scores: 0.66\n",+ "Average of max per entry across top 5 scores: 0.688\n",+ "Average of max per entry across top 8 scores: 0.692\n",+ "Average of max per entry across top 9999 scores: 0.692\n",+ "9 candidate programs found.\n",+ "[('retrieve', <dspy.retrieve.retrieve.Retrieve object at 0x12e43f670>), ('generate_query[0]', Predict(StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ "))), ('generate_query[1]', Predict(StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ "))), ('generate_answer', Predict(StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")))]\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ }+ ],"source": ["RECOMPILE_FT_MODEL = True\n","\n",@@ -309,19 +641,78 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 16,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[autoreload of dsp.modules.lm failed: Traceback (most recent call last):\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",+ " superreload(m, reload, self.old_objects)\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",+ " module = reload(module)\n",+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",+ " _bootstrap._exec(spec, module)\n",+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py\", line 152, in <module>\n",+ " class TrainableLM(LM, ABC):\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py\", line 153, in TrainableLM\n",+ " def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['TrainableLM']:\n",+ "NameError: name 'FinetuningMethod' is not defined\n",+ "]\n",+ "[autoreload of dsp.modules.openai failed: Traceback (most recent call last):\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",+ " superreload(m, reload, self.old_objects)\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",+ " module = reload(module)\n",+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",+ " _bootstrap._exec(spec, module)\n",+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py\", line 6, in <module>\n",+ " from dsp.modules.lm import TrainableLM, FinetuningMethod\n",+ "ImportError: cannot import name 'TrainableLM' from 'dsp.modules.lm' (/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py)\n",+ "]\n",+ "[autoreload of dspy failed: Traceback (most recent call last):\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",+ " superreload(m, reload, self.old_objects)\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",+ " module = reload(module)\n",+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",+ " _bootstrap._exec(spec, module)\n",+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dspy/__init__.py\", line 42, in <module>\n",+ " OpenAIModel = dsp.TrainableOpenAI\n",+ "AttributeError: module 'dsp' has no attribute 'TrainableOpenAI'\n",+ "]\n",+ "Average Metric: 130 / 300 (43.3): 100%|██████████| 300/300 [00:24<00:00, 12.29it/s] \n"+ ]+ },+ {+ "ename": "AttributeError",+ "evalue": "'float' object has no attribute '_get_lm_info_str'",+ "output_type": "error",+ "traceback": [+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",+ "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",+ "Cell \u001b[0;32mIn[16], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m TEST_SIZE \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[1;32m 2\u001b[0m baseline_eval \u001b[38;5;241m=\u001b[39m evaluate(BasicMH(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mprogram_params), devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[0;32m----> 3\u001b[0m \u001b[43mbaseline_eval\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_lm_info_str\u001b[49m()\n\u001b[1;32m 4\u001b[0m bs_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[1;32m 5\u001b[0m bs_eval\u001b[38;5;241m.\u001b[39m_get_lm_info_str()\n",+ "\u001b[0;31mAttributeError\u001b[0m: 'float' object has no attribute '_get_lm_info_str'"+ ]+ }+ ],"source": ["TEST_SIZE = 300\n","baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",- "baseline_eval._get_lm_info_str()\n","bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",- "bs_eval._get_lm_info_str()\n","bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",- "bs_ft_eval._get_lm_info_str()\n","bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",- "bs_ft_bs_eval._get_lm_info_str()\n","\n","print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to 1 attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n","print(f\"Non-finetuned model: {baseline_eval}\")\n",