Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Review changes

Isaac Millerb1ddecc2b08e
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 4b3fbec2d..42cf8222f 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -21,7 +21,7 @@ from .hf_client import Anyscale, HFClientTGI, Together
from .llama import *
from .mistral import *
from .ollama import *
-from .openai import *
+from .trainable_openai import *
from .multi_openai import MultiOpenAI
from .premai import PremAI
from .pyserini import *
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index a06e23236..9509288ea 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
from asyncio import Future
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
+from enum import Enum
from typing import Literal, Optional
class LM(ABC):
@@ -144,10 +145,13 @@ class LM(ABC):
return self.__class__(model=model, **kwargs)
-FinetuningMethod = Literal["SFT", "Contrastive"]
-class FinetunableLM(LM, ABC):
- def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLM']:
- future: Future['FinetunableLM'] = Future()
+class TrainingMethod(Enum):
+ SFT = "SFT"
+ Contrastive = "Contrastive"
+
+class TrainableLM(LM, ABC):
+ def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['TrainableLM']:
+ future: Future['TrainableLM'] = Future()
new_lm = deepcopy(self)
@@ -159,7 +163,7 @@ class FinetunableLM(LM, ABC):
return future
@abstractmethod
- def start_training(self, future: Future['FinetunableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):
+ def start_training(self, future: Future['TrainableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):
pass
@abstractmethod
diff --git a/dsp/modules/openai.py b/dsp/modules/trainable_openai.py
similarity index 90%
rename from dsp/modules/openai.py
rename to dsp/modules/trainable_openai.py
index 58abb381d..8535ff2fa 100644
--- a/dsp/modules/openai.py
+++ b/dsp/modules/trainable_openai.py
@@ -3,7 +3,7 @@ import time
from typing import Any, List, Optional, Literal, Union
import ujson
import openai
-from dsp.modules.lm import FinetunableLM, FinetuningMethod
+from dsp.modules.lm import TrainableLM, FinetuningMethod
from dsp.modules.gpt3 import GPT3
from collections import defaultdict
@@ -151,7 +151,7 @@ def backoff_hdlr(details):
"{kwargs}".format(**details),
)
-class OpenAIModel(GPT3, FinetunableLM):
+class TrainableOpenAI(GPT3, TrainableLM):
"""Wrapper around specifically the OpenAI API to finetune.
Args:
@@ -160,8 +160,6 @@ class OpenAIModel(GPT3, FinetunableLM):
api_provider (Literal["openai"], optional): The API provider to use. Defaults to "openai".
model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".
system_prompt (Optional[str], optional): The system prompt to use. Defaults to None in init, and "You are a helpful assistant." in format_data_for_vanilla_finetuning.
- fine_tuning_file_ids (dict, optional): The file ids of the training and validation data. Defaults to {}.
- fine_tuning_job_id (Optional[str], optional): The job id of the fine-tuning job. Defaults to None. Again, checkpointing is not supported ATM but this would be the place to store it.
**kwargs: Additional arguments to pass to the API provider.
"""
@@ -173,16 +171,12 @@ class OpenAIModel(GPT3, FinetunableLM):
api_base: Optional[str] = None,
model_type: Literal["chat", "text"] = None,
system_prompt: Optional[str] = None,
- fine_tuning_file_ids: dict = {},
- fine_tuning_job_id: Optional[str] = None,
**kwargs,
):
super().__init__(model, api_key=api_key, api_provider=api_provider, api_base=api_base, model_type=model_type, system_prompt=system_prompt, **kwargs)
assert self.provider == "openai", "You must use an OpenAI model with this class."
- self.fine_tuning_file_ids = fine_tuning_file_ids
- self.fine_tuning_job_id = fine_tuning_job_id
- def verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:
+ def _verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:
"""Verify the training arguments before starting training.
More information on dataset verification can be found here: https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset
@@ -222,7 +216,7 @@ class OpenAIModel(GPT3, FinetunableLM):
return True
- def format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, Any]]:
+ def _format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, Any]]:
def format_single_item(item):
messages = [{"role": "user", "content": item["prompt"]}, {
"role": "assistant", "content": item["completion"]}]
@@ -255,7 +249,7 @@ class OpenAIModel(GPT3, FinetunableLM):
self.fine_tuning_file_ids[name] = file.id
# TODO: support OAI wandb integration
- def start_remote_training(self, **kwargs) -> str:
+ def _start_remote_training(self, **kwargs) -> str:
assert self.fine_tuning_file_ids["train"] is not None, "You must load data before starting training"
hyperparameters = kwargs.get("hyperparameters", {})
job = openai.fine_tuning.jobs.create(
@@ -330,7 +324,7 @@ class OpenAIModel(GPT3, FinetunableLM):
else:
raise RuntimeError("Job not completed yet, cannot retrieve model")
- def start_training(self, future: Future['OpenAIModel'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):
+ def start_training(self, future: Future['TrainableOpenAI'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):
"""
Handles the fine-tuning process for an OpenAIModel instance.
@@ -345,15 +339,14 @@ class OpenAIModel(GPT3, FinetunableLM):
traindataset = ujson.load(open(train_path))
valdataset = ujson.load(open(val_path)) if val_path else None
- self.verify_training_arguments(traindataset, valdataset, kwargs)
+ self._verify_training_arguments(traindataset, valdataset, kwargs)
self._load_data(train_path, val_path)
# Start the remote training
- job_id = self.start_remote_training(**kwargs)
+ job_id = self._start_remote_training(**kwargs)
# Wait for the training to complete
- while not self.check_training_status():
- time.sleep(60) # Check every 60 seconds
+ self.wait_for_training()
# Retrieve the trained model and return a copy
self.retrieve_trained_model_client()
@@ -363,7 +356,6 @@ class OpenAIModel(GPT3, FinetunableLM):
future.set_exception(e)
def wait_for_training(self):
- # Wait for the future to complete
while not self.check_training_status():
time.sleep(60)
@@ -378,6 +370,6 @@ class OpenAIModel(GPT3, FinetunableLM):
job = openai.fine_tuning.jobs.retrieve(job_id)
if job.status != "succeeded":
raise RuntimeError("Job not completed yet, cannot retrieve model")
- model = OpenAIModel(model=job.fine_tuned_model, **kwargs)
+ model = TrainableOpenAI(model=job.fine_tuned_model, **kwargs)
model.fine_tuning_job_id = job_id
return model
\ No newline at end of file
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 7cba0e252..05d2a6d48 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -39,7 +39,7 @@ Anyscale = dsp.Anyscale
Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
-OpenAIModel = dsp.OpenAIModel
+TrainableOpenAI = dsp.TrainableOpenAI
LlamaCpp = dsp.LlamaCpp
Bedrock = dsp.Bedrock
diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynb
index cab90ad4c..bd9643bd7 100644
--- a/examples/finetuning/openai_finetuning_hpqa.ipynb
+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb
@@ -171,21 +171,21 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 257 / 500 (51.4): 100%|██████████| 500/500 [00:01<00:00, 342.65it/s]\n"
+ "Average Metric: 259 / 500 (51.8): 100%|██████████| 500/500 [00:01<00:00, 321.91it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Writing dataset with length 771 to trainset_data.jsonl\n"
+ "Writing dataset with length 777 to trainset_data.jsonl\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:43<00:00, 2.87it/s]"
+ "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:00<00:00, 350.95it/s]\n"
]
},
{
@@ -194,13 +194,6 @@
"text": [
"Writing dataset with length 228 to devset_data.jsonl\n"
]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n"
- ]
}
],
"source": [
@@ -242,34 +235,21 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 10,
"metadata": {},
- "outputs": [
- {
- "ename": "AttributeError",
- "evalue": "'_asyncio.Future' object has no attribute '_condition'",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m future_lm \u001b[38;5;241m=\u001b[39m lm\u001b[38;5;241m.\u001b[39mget_finetune(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrainset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdevset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSFT\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# finetuned_lm = future_lm.result()\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m finetuned_lm \u001b[38;5;241m=\u001b[39m \u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfuture_lm\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_when\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIRST_COMPLETED\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mresult() \n\u001b[1;32m 6\u001b[0m finetuned_lm\u001b[38;5;241m.\u001b[39mkwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:289\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(fs, timeout, return_when)\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait\u001b[39m(fs, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, return_when\u001b[38;5;241m=\u001b[39mALL_COMPLETED):\n\u001b[1;32m 266\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Wait for the futures in the given sequence to complete.\u001b[39;00m\n\u001b[1;32m 267\u001b[0m \n\u001b[1;32m 268\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;124;03m futures.\u001b[39;00m\n\u001b[1;32m 288\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _AcquireFutures(fs):\n\u001b[1;32m 290\u001b[0m done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m fs\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED_AND_NOTIFIED, FINISHED])\n\u001b[1;32m 292\u001b[0m not_done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(fs) \u001b[38;5;241m-\u001b[39m done\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:151\u001b[0m, in \u001b[0;36m_AcquireFutures.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__enter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 150\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfutures:\n\u001b[0;32m--> 151\u001b[0m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241m.\u001b[39macquire()\n",
- "\u001b[0;31mAttributeError\u001b[0m: '_asyncio.Future' object has no attribute '_condition'"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import concurrent.futures\n",
- "\n",
- "future_lm = lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",
- "finetuned_lm = await future_lm\n",
+ "# lm.()\n",
+ "# lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",
+ "future_lm = dspy.OpenAIModel.load_from_job_id(\"ftjob-tmccSJv170p4gNFJUYwXdCSq\")\n",
+ "finetuned_lm = future_lm\n",
"finetuned_lm.kwargs[\"temperature\"] = 0.0"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -278,7 +258,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -288,9 +268,361 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Going to sample between 1 and 3 traces per predictor.\n",
+ "Will attempt to bootstrap 6 candidate sets.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 106 / 250 (42.4): 100%|██████████| 250/250 [00:03<00:00, 71.56it/s] \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 42.4 for set: [0, 0, 0]\n",
+ "New best score: 42.4 for seed -3\n",
+ "Scores so far: [42.4]\n",
+ "Best score: 42.4\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 134 / 250 (53.6): 100%|██████████| 250/250 [00:29<00:00, 8.54it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 53.6 for set: [16, 16, 16]\n",
+ "New best score: 53.6 for seed -2\n",
+ "Scores so far: [42.4, 53.6]\n",
+ "Best score: 53.6\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 6%|▌ | 6/100 [00:26<07:02, 4.50s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 3 full traces after 7 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 89 / 162 (54.9): 64%|██████▍ | 161/250 [01:00<00:34, 2.62it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 4 / 8 (50.0): 3%|▎ | 8/250 [10:14<5:10:00, 76.86s/it]"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 90 / 163 (55.2): 65%|██████▌ | 163/250 [01:00<00:26, 3.30it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 142 / 250 (56.8): 100%|██████████| 250/250 [01:31<00:00, 2.74it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 56.8 for set: [16, 16, 16]\n",
+ "New best score: 56.8 for seed -1\n",
+ "Scores so far: [42.4, 53.6, 56.8]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.64\n",
+ "Average of max per entry across top 3 scores: 0.656\n",
+ "Average of max per entry across top 5 scores: 0.656\n",
+ "Average of max per entry across top 8 scores: 0.656\n",
+ "Average of max per entry across top 9999 scores: 0.656\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 3%|▎ | 3/100 [00:15<08:17, 5.13s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 2 full traces after 4 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 133 / 250 (53.2): 100%|██████████| 250/250 [01:49<00:00, 2.28it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 53.2 for set: [16, 16, 16]\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.64\n",
+ "Average of max per entry across top 3 scores: 0.664\n",
+ "Average of max per entry across top 5 scores: 0.676\n",
+ "Average of max per entry across top 8 scores: 0.676\n",
+ "Average of max per entry across top 9999 scores: 0.676\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [00:04<07:14, 4.39s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 136 / 250 (54.4): 100%|██████████| 250/250 [01:49<00:00, 2.29it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 54.4 for set: [16, 16, 16]\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.624\n",
+ "Average of max per entry across top 3 scores: 0.664\n",
+ "Average of max per entry across top 5 scores: 0.684\n",
+ "Average of max per entry across top 8 scores: 0.684\n",
+ "Average of max per entry across top 9999 scores: 0.684\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 2%|▏ | 2/100 [00:06<05:32, 3.39s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 135 / 250 (54.0): 100%|██████████| 250/250 [01:46<00:00, 2.34it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 54.0 for set: [16, 16, 16]\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.624\n",
+ "Average of max per entry across top 3 scores: 0.648\n",
+ "Average of max per entry across top 5 scores: 0.68\n",
+ "Average of max per entry across top 8 scores: 0.688\n",
+ "Average of max per entry across top 9999 scores: 0.688\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 2%|▏ | 2/100 [00:10<08:25, 5.16s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 125 / 250 (50.0): 100%|██████████| 250/250 [13:00<00:00, 3.12s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 50.0 for set: [16, 16, 16]\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.624\n",
+ "Average of max per entry across top 3 scores: 0.648\n",
+ "Average of max per entry across top 5 scores: 0.68\n",
+ "Average of max per entry across top 8 scores: 0.688\n",
+ "Average of max per entry across top 9999 scores: 0.688\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [17:22<28:40:26, 1042.69s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 133 / 250 (53.2): 100%|██████████| 250/250 [01:35<00:00, 2.61it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 53.2 for set: [16, 16, 16]\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0, 53.2]\n",
+ "Best score: 56.8\n",
+ "Average of max per entry across top 1 scores: 0.568\n",
+ "Average of max per entry across top 2 scores: 0.624\n",
+ "Average of max per entry across top 3 scores: 0.648\n",
+ "Average of max per entry across top 5 scores: 0.68\n",
+ "Average of max per entry across top 8 scores: 0.688\n",
+ "Average of max per entry across top 9999 scores: 0.688\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 8%|▊ | 8/100 [00:44<08:35, 5.60s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 3 full traces after 9 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 143 / 250 (57.2): 100%|██████████| 250/250 [01:47<00:00, 2.33it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 57.2 for set: [16, 16, 16]\n",
+ "New best score: 57.2 for seed 5\n",
+ "Scores so far: [42.4, 53.6, 56.8, 53.2, 54.4, 54.0, 50.0, 53.2, 57.2]\n",
+ "Best score: 57.2\n",
+ "Average of max per entry across top 1 scores: 0.572\n",
+ "Average of max per entry across top 2 scores: 0.64\n",
+ "Average of max per entry across top 3 scores: 0.66\n",
+ "Average of max per entry across top 5 scores: 0.688\n",
+ "Average of max per entry across top 8 scores: 0.692\n",
+ "Average of max per entry across top 9999 scores: 0.692\n",
+ "9 candidate programs found.\n",
+ "[('retrieve', <dspy.retrieve.retrieve.Retrieve object at 0x12e43f670>), ('generate_query[0]', Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ "))), ('generate_query[1]', Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ "))), ('generate_answer', Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ ")))]\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
"source": [
"RECOMPILE_FT_MODEL = True\n",
"\n",
@@ -309,19 +641,78 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[autoreload of dsp.modules.lm failed: Traceback (most recent call last):\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
+ " superreload(m, reload, self.old_objects)\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
+ " module = reload(module)\n",
+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",
+ " _bootstrap._exec(spec, module)\n",
+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",
+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",
+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py\", line 152, in <module>\n",
+ " class TrainableLM(LM, ABC):\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py\", line 153, in TrainableLM\n",
+ " def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['TrainableLM']:\n",
+ "NameError: name 'FinetuningMethod' is not defined\n",
+ "]\n",
+ "[autoreload of dsp.modules.openai failed: Traceback (most recent call last):\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
+ " superreload(m, reload, self.old_objects)\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
+ " module = reload(module)\n",
+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",
+ " _bootstrap._exec(spec, module)\n",
+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",
+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",
+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py\", line 6, in <module>\n",
+ " from dsp.modules.lm import TrainableLM, FinetuningMethod\n",
+ "ImportError: cannot import name 'TrainableLM' from 'dsp.modules.lm' (/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dsp/modules/lm.py)\n",
+ "]\n",
+ "[autoreload of dspy failed: Traceback (most recent call last):\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
+ " superreload(m, reload, self.old_objects)\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
+ " module = reload(module)\n",
+ " File \"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\", line 169, in reload\n",
+ " _bootstrap._exec(spec, module)\n",
+ " File \"<frozen importlib._bootstrap>\", line 613, in _exec\n",
+ " File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",
+ " File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",
+ " File \"/Users/isaac.miller/projects/dspy-finetuning/dilara-dspy/dspy/__init__.py\", line 42, in <module>\n",
+ " OpenAIModel = dsp.TrainableOpenAI\n",
+ "AttributeError: module 'dsp' has no attribute 'TrainableOpenAI'\n",
+ "]\n",
+ "Average Metric: 130 / 300 (43.3): 100%|██████████| 300/300 [00:24<00:00, 12.29it/s] \n"
+ ]
+ },
+ {
+ "ename": "AttributeError",
+ "evalue": "'float' object has no attribute '_get_lm_info_str'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[16], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m TEST_SIZE \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[1;32m 2\u001b[0m baseline_eval \u001b[38;5;241m=\u001b[39m evaluate(BasicMH(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mprogram_params), devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[0;32m----> 3\u001b[0m \u001b[43mbaseline_eval\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_lm_info_str\u001b[49m()\n\u001b[1;32m 4\u001b[0m bs_eval \u001b[38;5;241m=\u001b[39m evaluate(basicmh_bs, devset\u001b[38;5;241m=\u001b[39mdevset[:TEST_SIZE])\n\u001b[1;32m 5\u001b[0m bs_eval\u001b[38;5;241m.\u001b[39m_get_lm_info_str()\n",
+ "\u001b[0;31mAttributeError\u001b[0m: 'float' object has no attribute '_get_lm_info_str'"
+ ]
+ }
+ ],
"source": [
"TEST_SIZE = 300\n",
"baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",
- "baseline_eval._get_lm_info_str()\n",
"bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",
- "bs_eval._get_lm_info_str()\n",
"bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",
- "bs_ft_eval._get_lm_info_str()\n",
"bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",
- "bs_ft_bs_eval._get_lm_info_str()\n",
"\n",
"print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to 1 attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n",
"print(f\"Non-finetuned model: {baseline_eval}\")\n",