Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Implement LM Future

Isaac Millere08752636bf2
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index f4aa1d876..320432fdd 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -148,59 +148,25 @@ class LM(ABC):
model = kwargs.pop("model")
return self.__class__(model=model, **kwargs)
-
-class FinetuneableLM(LM, ABC):
- def verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:
- return True
-
- @abstractmethod
- def format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, str]]:
- pass
- @abstractmethod
- def _load_data(self, train_path: str, val_path: Optional[str]) -> str:
- pass
-
- @abstractmethod
- def start_training(self, **kwargs) -> None:
- pass
-
- @abstractmethod
- def stop_training(self) -> None:
- pass
-
- @abstractmethod
- def check_training_status(self) -> bool:
- pass
-
- @abstractmethod
- def retrieve_trained_model_client(self) -> LM:
- pass
+FinetuningMethod = Literal["SFT", "Contrastive"]
+class FinetunableLM(LM, ABC):
+ def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLM']:
+ future: Future['FinetunableLM'] = Future()
+
+ new_lm = deepcopy(self)
-FinetuningMethod = Literal["SFT", "Contrastive"]
-class FinetunableLMFuture(LM, ABC):
- def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLMFuture']:
- future: Future['FinetunableLMFuture'] = Future()
-
- def run_finetune(original: 'FinetunableLMFuture'):
- try:
- copy = deepcopy(original)
- result = copy.start_training(train_path, val_path, method, **kwargs)
- future.set_result(result)
- except Exception as e:
- future.set_exception(e)
-
+ # Capture the current instance in the closure and start the training process asynchronously
executor = ThreadPoolExecutor(max_workers=1)
-
- executor.submit(run_finetune, self)
+ executor.submit(new_lm.start_training, future, train_path=train_path, val_path=val_path, method=method, **kwargs)
executor.shutdown(wait=False)
return future
@abstractmethod
- def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':
+ def start_training(self, future: Future['FinetunableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):
pass
@abstractmethod
- def check_status(self) -> Optional[str]:
+ def stop_training(self) -> None:
pass
\ No newline at end of file
diff --git a/dsp/modules/openai.py b/dsp/modules/openai.py
index 4028f57cf..87c7d10d8 100644
--- a/dsp/modules/openai.py
+++ b/dsp/modules/openai.py
@@ -1,9 +1,10 @@
+from concurrent.futures import Future, ThreadPoolExecutor
from copy import deepcopy
import time
from typing import Any, Optional, Literal, Union
import openai
-from dsp.modules.lm import LM, FinetuneableLM
+from dsp.modules.lm import LM, FinetunableLM
from dsp.modules.gpt3 import GPT3
import tiktoken
@@ -150,7 +151,7 @@ def backoff_hdlr(details):
FinetuningMethod = Literal["SFT", "Contrastive"]
-class OpenAIModel(GPT3, FinetuneableLM):
+class OpenAIModel(GPT3, FinetunableLM):
"""Wrapper around specifically the OpenAI API to finetune.
Args:
@@ -320,30 +321,57 @@ class OpenAIModel(GPT3, FinetuneableLM):
elif temp_job.status == "running":
return False
- def retrieve_trained_model_client(self) -> LM:
+ def retrieve_trained_model_client(self):
assert self.fine_tuning_job_id is not None, "Start training before retrieving the model"
job = openai.fine_tuning.jobs.retrieve(self.fine_tuning_job_id)
if job.status == "succeeded":
- self_copy = deepcopy(self)
- self_copy.kwargs["model"] = job.fine_tuned_model
- return self_copy
+ # NOTE: Not making a copy here because that is done before the training process starts
+ self.kwargs["model"] = job.fine_tuned_model
else:
raise RuntimeError("Job not completed yet, cannot retrieve model")
-
- def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':
- # Maybe validate data, hparams here
- self._load_data(train_path, val_path)
- if method != "SFT":
- raise NotImplementedError("Only SFT finetuning is supported at the moment.")
- self.start_remote_training(**kwargs)
- self.wait_for_training()
+
+ def start_training(self, future: Future['OpenAIModel'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):
+ """
+ Handles the fine-tuning process for an OpenAIModel instance.
+
+ Args:
+ original_model: The original model instance to be fine-tuned.
+ future: The Future object that will hold the fine-tuned model.
+ **kwargs: Additional arguments for fine-tuning.
+ """
+ try:
+ if method != "SFT":
+ raise NotImplementedError("Only SFT finetuning is supported at the moment.")
+
+ self._load_data(train_path, val_path)
+ # Start the remote training
+ job_id = self.start_remote_training(**kwargs)
+
+ # Wait for the training to complete
+ while not self.check_training_status():
+ time.sleep(60) # Check every 60 seconds
- # Return a future that can be used to check the status of the training
+ # Retrieve the trained model and return a copy
+ self.retrieve_trained_model_client()
+ future.set_result(self)
+
+ except Exception as e:
+ future.set_exception(e)
def wait_for_training(self):
# Wait for the future to complete
while not self.check_training_status():
time.sleep(60)
- def check_status(self) -> Optional[str]:
- pass
+ @staticmethod
+ def load_from_job_id(job_id: str, **kwargs):
+ """Load a model from a job id.
+
+ Args:
+ job_id: The job id of the fine-tuning job.
+ **kwargs: Additional arguments to pass to the API provider.
+ """
+ job = openai.fine_tuning.jobs.retrieve(job_id)
+ model = OpenAIModel(model=job.fine_tuned_model, **kwargs)
+ model.fine_tuning_job_id = job_id
+ return model
\ No newline at end of file
diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynb
index 5bf645e99..cab90ad4c 100644
--- a/examples/finetuning/openai_finetuning_hpqa.ipynb
+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb
@@ -76,31 +76,6 @@
"source": [
"import dspy.evaluate\n",
"\n",
- "\n",
- "class AnswerCorrectnessSignature(dspy.Signature):\n",
- " \"\"\"Verify that the predicted answer matches the gold answer.\"\"\"\n",
- "\n",
- " question = dspy.InputField()\n",
- " gold_answer = dspy.InputField(desc=\"correct answer for question\")\n",
- " predicted_answer = dspy.InputField(desc=\"predicted answer for question\")\n",
- " is_correct = dspy.OutputField(desc='True or False')\n",
- "\n",
- "class AnswerCorrectness(dspy.Module):\n",
- " def __init__(self):\n",
- " super().__init__()\n",
- " self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)\n",
- " \n",
- " def forward(self, example, predicted_answer):\n",
- " question, gold_answer = example.question, example.answer\n",
- " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=100, temperature=0.1)):\n",
- " return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)\n",
- " \n",
- "evaluator = AnswerCorrectness()\n",
- "\n",
- "# def metric(example, pred, trace=None):\n",
- "# result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",
- "# return \"true\" in result and not \"false\" in result\n",
- "\n",
"metric = dspy.evaluate.answer_exact_match"
]
},
@@ -164,16 +139,6 @@
"execution_count": 8,
"metadata": {},
"outputs": [],
- "source": [
- "# program = BasicMH()\n",
- "# program(question=\"What is the capital of France?\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [],
"source": [
"from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",
"\n",
@@ -199,35 +164,42 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 62 / 100 (62.0): 100%|██████████| 100/100 [00:00<00:00, 351.07it/s]\n"
+ "Average Metric: 257 / 500 (51.4): 100%|██████████| 500/500 [00:01<00:00, 342.65it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "186\n"
+ "Writing dataset with length 771 to trainset_data.jsonl\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 343.97it/s]\n"
+ "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:43<00:00, 2.87it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "168\n"
+ "Writing dataset with length 228 to devset_data.jsonl\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
]
}
],
@@ -237,9 +209,9 @@
"from typing import Callable\n",
"import ujson\n",
"\n",
- "samples = 100\n",
+ "samples = 500\n",
"\n",
- "callback = DataCollectionCallback(num_correct=1, max_attempts=1)\n",
+ "callback = DataCollectionCallback(num_correct=1, max_attempts=2)\n",
"\n",
"dspy.settings.configure(experimental=True)\n",
"dc_kwargs = {\n",
@@ -249,94 +221,50 @@
" \"next_round_dataset_callback\": callback.move_on_callback_correct_with_max,\n",
" \"num_threads\": NUM_THREADS,\n",
"}\n",
- "basicmh_bs = BasicMH(**program_params)\n",
- "basicmh_bs.load(\"basicmh_3_3_6.json\")\n",
- "# NOTE: It looks like all data, not just success is saved\n",
"\n",
- "trainset_data = bootstrap_data_multiple_rounds(basicmh_bs, trainset[:samples], metric, num_rounds=1, **dc_kwargs)\n",
- "trainset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in trainset_data if result[\"score\"]], [])\n",
- "trainset_data = lm.format_data_for_vanilla_finetuning(trainset_data)\n",
- "print(len(trainset_data))\n",
- "ts_json = ujson.dumps(trainset_data)\n",
- "with open(\"trainset_data.jsonl\", \"w\") as f:\n",
- " for line in trainset_data:\n",
- " f.write(ujson.dumps(line) + \"\\n\")\n",
- "\n",
- "devset_data = bootstrap_data_multiple_rounds(basicmh_bs, devset[:int(samples)], metric, num_rounds=1, **dc_kwargs)\n",
- "devset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in devset_data if result[\"score\"]], [])\n",
- "devset_data = lm.format_data_for_vanilla_finetuning(devset_data)\n",
- "print(len(devset_data))\n",
- "ds_json = ujson.dumps(devset_data)\n",
- "with open(\"devset_data.jsonl\", \"w\") as f:\n",
- " for line in devset_data:\n",
- " f.write(ujson.dumps(line) + \"\\n\")"
+ "dataset_filenames = {\"trainset_data.jsonl\": trainset[:samples], \"devset_data.jsonl\": devset[:int(samples/4)]}\n",
+ "\n",
+ "def write_data(data, filename):\n",
+ " # get the bootstrapped data for num_rounds=1, but using the callback\n",
+ " dataset = bootstrap_data_multiple_rounds(basicmh_bs, data, metric, num_rounds=1, **dc_kwargs)\n",
+ " # Flatmap the lists of prompt completions gives by each trace\n",
+ " dataset = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in dataset if result[\"score\"]], [])\n",
+ " # Format the data for finetuning using the LM\n",
+ " dataset = lm.format_data_for_vanilla_finetuning(dataset)\n",
+ " print(\"Writing dataset with length\", len(dataset), \"to\", filename)\n",
+ " with open(filename, \"w\") as f:\n",
+ " for line in dataset:\n",
+ " f.write(ujson.dumps(line) + \"\\n\")\n",
+ "\n",
+ "for key, data in dataset_filenames.items():\n",
+ " write_data(data, key)"
]
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
- "ename": "APIConnectionError",
- "evalue": "Connection error.",
+ "ename": "AttributeError",
+ "evalue": "'_asyncio.Future' object has no attribute '_condition'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mConnectError\u001b[0m Traceback (most recent call last)",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:69\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[0;34m()\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 69\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:233\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 233\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mIterable)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py:216\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_connections(closing)\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py:196\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 195\u001b[0m \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[0;32m--> 196\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_request\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[1;32m 200\u001b[0m \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[1;32m 201\u001b[0m \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[1;32m 202\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:99\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connect_failed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 99\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection\u001b[38;5;241m.\u001b[39mhandle_request(request)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:76\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 76\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m ssl_object \u001b[38;5;241m=\u001b[39m stream\u001b[38;5;241m.\u001b[39mget_extra_info(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mssl_object\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:122\u001b[0m, in \u001b[0;36mHTTPConnection._connect\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconnect_tcp\u001b[39m\u001b[38;5;124m\"\u001b[39m, logger, request, kwargs) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[0;32m--> 122\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_network_backend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect_tcp\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 123\u001b[0m trace\u001b[38;5;241m.\u001b[39mreturn_value \u001b[38;5;241m=\u001b[39m stream\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_backends/sync.py:213\u001b[0m, in \u001b[0;36mSyncBackend.connect_tcp\u001b[0;34m(self, host, port, timeout, local_address, socket_options)\u001b[0m\n\u001b[1;32m 212\u001b[0m sock\u001b[38;5;241m.\u001b[39msetsockopt(\u001b[38;5;241m*\u001b[39moption) \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m sock\u001b[38;5;241m.\u001b[39msetsockopt(socket\u001b[38;5;241m.\u001b[39mIPPROTO_TCP, socket\u001b[38;5;241m.\u001b[39mTCP_NODELAY, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SyncStream(sock)\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py:135\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgen\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mthrow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraceback\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;66;03m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[38;5;66;03m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_exceptions.py:14\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[0;34m(map)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(exc, from_exc):\n\u001b[0;32m---> 14\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m to_exc(exc) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n",
- "\u001b[0;31mConnectError\u001b[0m: [Errno 8] nodename nor servname provided, or not known",
- "\nThe above exception was the direct cause of the following exception:\n",
- "\u001b[0;31mConnectError\u001b[0m Traceback (most recent call last)",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:978\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 977\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 978\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 979\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 980\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_should_stream_response_body\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 981\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 982\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 983\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mTimeoutException \u001b[38;5;28;01mas\u001b[39;00m err:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:914\u001b[0m, in \u001b[0;36mClient.send\u001b[0;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[1;32m 912\u001b[0m auth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_request_auth(request, auth)\n\u001b[0;32m--> 914\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_auth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 915\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 916\u001b[0m \u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 917\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 918\u001b[0m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 919\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 920\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:942\u001b[0m, in \u001b[0;36mClient._send_handling_auth\u001b[0;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 942\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_redirects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 943\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 945\u001b[0m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhistory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 946\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 947\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:979\u001b[0m, in \u001b[0;36mClient._send_handling_redirects\u001b[0;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[1;32m 977\u001b[0m hook(request)\n\u001b[0;32m--> 979\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_single_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 980\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:1015\u001b[0m, in \u001b[0;36mClient._send_single_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 1014\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request\u001b[38;5;241m=\u001b[39mrequest):\n\u001b[0;32m-> 1015\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mtransport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1017\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, SyncByteStream)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:233\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 233\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool\u001b[38;5;241m.\u001b[39mhandle_request(req)\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mIterable)\n",
- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py:135\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgen\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mthrow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraceback\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;66;03m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[38;5;66;03m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:86\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[0;34m()\u001b[0m\n\u001b[1;32m 85\u001b[0m message \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(exc)\n\u001b[0;32m---> 86\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m mapped_exc(message) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",
- "\u001b[0;31mConnectError\u001b[0m: [Errno 8] nodename nor servname provided, or not known",
- "\nThe above exception was the direct cause of the following exception:\n",
- "\u001b[0;31mAPIConnectionError\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mlm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart_training\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtrainset_data.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdevset_data.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSFT\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:339\u001b[0m, in \u001b[0;36mOpenAIModel.start_training\u001b[0;34m(self, train_path, val_path, method, **kwargs)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOnly SFT finetuning is supported at the moment.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 338\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstart_remote_training(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m--> 339\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_training\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:345\u001b[0m, in \u001b[0;36mOpenAIModel.wait_for_training\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait_for_training\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 344\u001b[0m \u001b[38;5;66;03m# Wait for the future to complete\u001b[39;00m\n\u001b[0;32m--> 345\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_training_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 346\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m60\u001b[39m)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:313\u001b[0m, in \u001b[0;36mOpenAIModel.check_training_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_training_status\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfine_tuning_job_id \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou must start training before checking status\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 313\u001b[0m temp_job \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfine_tuning\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjobs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfine_tuning_job_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m temp_job\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msucceeded\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 315\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/resources/fine_tuning/jobs/jobs.py:180\u001b[0m, in \u001b[0;36mJobs.retrieve\u001b[0;34m(self, fine_tuning_job_id, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fine_tuning_job_id:\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected a non-empty value for `fine_tuning_job_id` but received \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfine_tuning_job_id\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 180\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/fine_tuning/jobs/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mfine_tuning_job_id\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 182\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmake_request_options\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 183\u001b[0m \u001b[43m \u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_headers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_body\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\n\u001b[1;32m 184\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 185\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mFineTuningJob\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1209\u001b[0m, in \u001b[0;36mSyncAPIClient.get\u001b[0;34m(self, path, cast_to, options, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1206\u001b[0m opts \u001b[38;5;241m=\u001b[39m FinalRequestOptions\u001b[38;5;241m.\u001b[39mconstruct(method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mget\u001b[39m\u001b[38;5;124m\"\u001b[39m, url\u001b[38;5;241m=\u001b[39mpath, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions)\n\u001b[1;32m 1207\u001b[0m \u001b[38;5;66;03m# cast is required because mypy complains about returning Any even though\u001b[39;00m\n\u001b[1;32m 1208\u001b[0m \u001b[38;5;66;03m# it understands the type variables\u001b[39;00m\n\u001b[0;32m-> 1209\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m cast(ResponseT, \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m)\u001b[49m)\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:942\u001b[0m, in \u001b[0;36mSyncAPIClient.request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 933\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m 934\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 935\u001b[0m cast_to: Type[ResponseT],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 940\u001b[0m stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 941\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[0;32m--> 942\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 943\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 945\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 946\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 947\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 948\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1002\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 999\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered Exception\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 1001\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retries \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1004\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1005\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1006\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1007\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1008\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1009\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1079\u001b[0m, in \u001b[0;36mSyncAPIClient._retry_request\u001b[0;34m(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;66;03m# In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\u001b[39;00m\n\u001b[1;32m 1076\u001b[0m \u001b[38;5;66;03m# different thread if necessary.\u001b[39;00m\n\u001b[1;32m 1077\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(timeout)\n\u001b[0;32m-> 1079\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1080\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1081\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1082\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1083\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1084\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1085\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1002\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 999\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered Exception\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 1001\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retries \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1004\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1005\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1006\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1007\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1008\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1009\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1079\u001b[0m, in \u001b[0;36mSyncAPIClient._retry_request\u001b[0;34m(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;66;03m# In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\u001b[39;00m\n\u001b[1;32m 1076\u001b[0m \u001b[38;5;66;03m# different thread if necessary.\u001b[39;00m\n\u001b[1;32m 1077\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(timeout)\n\u001b[0;32m-> 1079\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1080\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1081\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1082\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1083\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1084\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1085\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1012\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_retry_request(\n\u001b[1;32m 1003\u001b[0m input_options,\n\u001b[1;32m 1004\u001b[0m cast_to,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1008\u001b[0m response_headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1009\u001b[0m )\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m 1014\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\n\u001b[1;32m 1015\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mHTTP Response: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%i\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 1016\u001b[0m request\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1020\u001b[0m response\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1022\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest_id: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, response\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mx-request-id\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n",
- "\u001b[0;31mAPIConnectionError\u001b[0m: Connection error."
+ "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m future_lm \u001b[38;5;241m=\u001b[39m lm\u001b[38;5;241m.\u001b[39mget_finetune(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrainset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdevset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSFT\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# finetuned_lm = future_lm.result()\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m finetuned_lm \u001b[38;5;241m=\u001b[39m \u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfuture_lm\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_when\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIRST_COMPLETED\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mresult() \n\u001b[1;32m 6\u001b[0m finetuned_lm\u001b[38;5;241m.\u001b[39mkwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n",
+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:289\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(fs, timeout, return_when)\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait\u001b[39m(fs, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, return_when\u001b[38;5;241m=\u001b[39mALL_COMPLETED):\n\u001b[1;32m 266\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Wait for the futures in the given sequence to complete.\u001b[39;00m\n\u001b[1;32m 267\u001b[0m \n\u001b[1;32m 268\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;124;03m futures.\u001b[39;00m\n\u001b[1;32m 288\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _AcquireFutures(fs):\n\u001b[1;32m 290\u001b[0m done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m fs\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED_AND_NOTIFIED, FINISHED])\n\u001b[1;32m 292\u001b[0m not_done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(fs) \u001b[38;5;241m-\u001b[39m done\n",
+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:151\u001b[0m, in \u001b[0;36m_AcquireFutures.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__enter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 150\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfutures:\n\u001b[0;32m--> 151\u001b[0m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241m.\u001b[39macquire()\n",
+ "\u001b[0;31mAttributeError\u001b[0m: '_asyncio.Future' object has no attribute '_condition'"
]
}
],
"source": [
- "lm.start_training(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",
- "# ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\n",
- "# teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"
+ "import concurrent.futures\n",
+ "\n",
+ "future_lm = lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",
+ "finetuned_lm = await future_lm\n",
+ "finetuned_lm.kwargs[\"temperature\"] = 0.0"
]
},
{
@@ -345,25 +273,7 @@
"metadata": {},
"outputs": [],
"source": [
- "TRAIN_FROM_SCRATCH = False\n",
- "\n",
- "if TRAIN_FROM_SCRATCH:\n",
- " teacher = basicmh_bs\n",
- " \n",
- " # something like\n",
- " # Go through all the possible methods + save/load\n",
- " # change to future[lm] first\n",
- "\n",
- "\n",
- " # TODO: Working here\n",
- " # teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)\n",
- "# kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",
- "# # NOTE: LM IS NOT SET PROPERLY\n",
- "# basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",
- "# basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",
- "# else:\n",
- "# basicmh_bs_ft = BasicMH(**program_params)\n",
- "# basicmh_bs_ft.load('mini_ft_hpqa_100.json')"
+ "assert finetuned_lm.kwargs[\"model\"] != lm.kwargs[\"model\"]"
]
},
{
@@ -372,13 +282,8 @@
"metadata": {},
"outputs": [],
"source": [
- "lm2 = lm.copy()\n",
- "lm2.kwargs[\"model\"] = \"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\"\n",
- "\n",
"basicmh_bs_ft = BasicMH(**program_params)\n",
- "basicmh_bs_ft._set_all_predictor_lms(lm2)\n",
- "# print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",
- "# dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"
+ "basicmh_bs_ft._set_all_predictor_lms(finetuned_lm)"
]
},
{
@@ -387,7 +292,7 @@
"metadata": {},
"outputs": [],
"source": [
- "RECOMPILE_FT_MODEL = False\n",
+ "RECOMPILE_FT_MODEL = True\n",
"\n",
"if RECOMPILE_FT_MODEL:\n",
" max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",
@@ -395,9 +300,11 @@
" bsfsrs_teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",
" basicmh_bs_ft_bs = bsfsrs_teleprompter.compile(student=basicmh_bs_ft, trainset=trainset[:100], valset=devset[:250])\n",
" basicmh_bs_ft_bs.save('mini_bs_ft_bs_hpqa_100.json')\n",
+ " basicmh_bs_ft_bs._set_all_predictor_lms(finetuned_lm)\n",
"else:\n",
" basicmh_bs_ft_bs = BasicMH(**program_params)\n",
- " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')"
+ " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')\n",
+ " basicmh_bs_ft_bs._set_all_predictor_lms(finetuned_lm)"
]
},
{
@@ -407,13 +314,14 @@
"outputs": [],
"source": [
"TEST_SIZE = 300\n",
- "\n",
- "with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",
- " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",
- " bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",
- "with dspy.context(lm=dspy.OpenAI(model=\"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\", max_tokens=500, temperature=0.0)):\n",
- " bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",
- " bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",
+ "baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",
+ "baseline_eval._get_lm_info_str()\n",
+ "bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",
+ "bs_eval._get_lm_info_str()\n",
+ "bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",
+ "bs_ft_eval._get_lm_info_str()\n",
+ "bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",
+ "bs_ft_bs_eval._get_lm_info_str()\n",
"\n",
"print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to 1 attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n",
"print(f\"Non-finetuned model: {baseline_eval}\")\n",