DSPy: The framework for programming—not prompting—language models
Python
Implement LM Future
Isaac Millere08752636bf2
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex f4aa1d876..320432fdd 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -148,59 +148,25 @@ class LM(ABC):model = kwargs.pop("model")return self.__class__(model=model, **kwargs)--class FinetuneableLM(LM, ABC):- def verify_training_arguments(self, dataset: dict[str, Any], valset: Optional[dict[str, Any]], training_arguments: dict[str, Any]) -> bool:- return True-- @abstractmethod- def format_data_for_vanilla_finetuning(self, data: list[dict[str, str]]) -> list[dict[str, str]]:- pass- @abstractmethod- def _load_data(self, train_path: str, val_path: Optional[str]) -> str:- pass-- @abstractmethod- def start_training(self, **kwargs) -> None:- pass-- @abstractmethod- def stop_training(self) -> None:- pass-- @abstractmethod- def check_training_status(self) -> bool:- pass-- @abstractmethod- def retrieve_trained_model_client(self) -> LM:- pass+FinetuningMethod = Literal["SFT", "Contrastive"]+class FinetunableLM(LM, ABC):+ def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLM']:+ future: Future['FinetunableLM'] = Future()++ new_lm = deepcopy(self)-FinetuningMethod = Literal["SFT", "Contrastive"]-class FinetunableLMFuture(LM, ABC):- def get_finetune(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> Future['FinetunableLMFuture']:- future: Future['FinetunableLMFuture'] = Future()-- def run_finetune(original: 'FinetunableLMFuture'):- try:- copy = deepcopy(original)- result = copy.start_training(train_path, val_path, method, **kwargs)- future.set_result(result)- except Exception as e:- future.set_exception(e)-+ # Capture the current instance in the closure and start the training process asynchronouslyexecutor = ThreadPoolExecutor(max_workers=1)-- executor.submit(run_finetune, self)+ executor.submit(new_lm.start_training, future, train_path=train_path, val_path=val_path, method=method, **kwargs)executor.shutdown(wait=False)return future@abstractmethod- def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':+ def start_training(self, future: Future['FinetunableLM'], train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs):pass@abstractmethod- def check_status(self) -> Optional[str]:+ def stop_training(self) -> None:pass\ No newline at end of filediff --git a/dsp/modules/openai.py b/dsp/modules/openai.pyindex 4028f57cf..87c7d10d8 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/openai.py@@ -1,9 +1,10 @@+from concurrent.futures import Future, ThreadPoolExecutorfrom copy import deepcopyimport timefrom typing import Any, Optional, Literal, Unionimport openai-from dsp.modules.lm import LM, FinetuneableLM+from dsp.modules.lm import LM, FinetunableLMfrom dsp.modules.gpt3 import GPT3import tiktoken@@ -150,7 +151,7 @@ def backoff_hdlr(details):FinetuningMethod = Literal["SFT", "Contrastive"]-class OpenAIModel(GPT3, FinetuneableLM):+class OpenAIModel(GPT3, FinetunableLM):"""Wrapper around specifically the OpenAI API to finetune.Args:@@ -320,30 +321,57 @@ class OpenAIModel(GPT3, FinetuneableLM):elif temp_job.status == "running":return False- def retrieve_trained_model_client(self) -> LM:+ def retrieve_trained_model_client(self):assert self.fine_tuning_job_id is not None, "Start training before retrieving the model"job = openai.fine_tuning.jobs.retrieve(self.fine_tuning_job_id)if job.status == "succeeded":- self_copy = deepcopy(self)- self_copy.kwargs["model"] = job.fine_tuned_model- return self_copy+ # NOTE: Not making a copy here because that is done before the training process starts+ self.kwargs["model"] = job.fine_tuned_modelelse:raise RuntimeError("Job not completed yet, cannot retrieve model")-- def start_training(self, train_path: str, val_path: Optional[str], method: FinetuningMethod, **kwargs) -> 'FinetunableLMFuture':- # Maybe validate data, hparams here- self._load_data(train_path, val_path)- if method != "SFT":- raise NotImplementedError("Only SFT finetuning is supported at the moment.")- self.start_remote_training(**kwargs)- self.wait_for_training()++ def start_training(self, future: Future['OpenAIModel'], train_path: str, val_path: Optional[str], method: FinetuningMethod = "SFT", **kwargs):+ """+ Handles the fine-tuning process for an OpenAIModel instance.++ Args:+ original_model: The original model instance to be fine-tuned.+ future: The Future object that will hold the fine-tuned model.+ **kwargs: Additional arguments for fine-tuning.+ """+ try:+ if method != "SFT":+ raise NotImplementedError("Only SFT finetuning is supported at the moment.")++ self._load_data(train_path, val_path)+ # Start the remote training+ job_id = self.start_remote_training(**kwargs)++ # Wait for the training to complete+ while not self.check_training_status():+ time.sleep(60) # Check every 60 seconds- # Return a future that can be used to check the status of the training+ # Retrieve the trained model and return a copy+ self.retrieve_trained_model_client()+ future.set_result(self)++ except Exception as e:+ future.set_exception(e)def wait_for_training(self):# Wait for the future to completewhile not self.check_training_status():time.sleep(60)- def check_status(self) -> Optional[str]:- pass+ @staticmethod+ def load_from_job_id(job_id: str, **kwargs):+ """Load a model from a job id.++ Args:+ job_id: The job id of the fine-tuning job.+ **kwargs: Additional arguments to pass to the API provider.+ """+ job = openai.fine_tuning.jobs.retrieve(job_id)+ model = OpenAIModel(model=job.fine_tuned_model, **kwargs)+ model.fine_tuning_job_id = job_id+ return model\ No newline at end of filediff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynbindex 5bf645e99..cab90ad4c 100644--- a/examples/finetuning/openai_finetuning_hpqa.ipynb+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb@@ -76,31 +76,6 @@"source": ["import dspy.evaluate\n","\n",- "\n",- "class AnswerCorrectnessSignature(dspy.Signature):\n",- " \"\"\"Verify that the predicted answer matches the gold answer.\"\"\"\n",- "\n",- " question = dspy.InputField()\n",- " gold_answer = dspy.InputField(desc=\"correct answer for question\")\n",- " predicted_answer = dspy.InputField(desc=\"predicted answer for question\")\n",- " is_correct = dspy.OutputField(desc='True or False')\n",- "\n",- "class AnswerCorrectness(dspy.Module):\n",- " def __init__(self):\n",- " super().__init__()\n",- " self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)\n",- " \n",- " def forward(self, example, predicted_answer):\n",- " question, gold_answer = example.question, example.answer\n",- " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=100, temperature=0.1)):\n",- " return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)\n",- " \n",- "evaluator = AnswerCorrectness()\n",- "\n",- "# def metric(example, pred, trace=None):\n",- "# result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",- "# return \"true\" in result and not \"false\" in result\n",- "\n","metric = dspy.evaluate.answer_exact_match"]},@@ -164,16 +139,6 @@"execution_count": 8,"metadata": {},"outputs": [],- "source": [- "# program = BasicMH()\n",- "# program(question=\"What is the capital of France?\")"- ]- },- {- "cell_type": "code",- "execution_count": 9,- "metadata": {},- "outputs": [],"source": ["from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n","\n",@@ -199,35 +164,42 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": 9,"metadata": {},"outputs": [{"name": "stderr","output_type": "stream","text": [- "Average Metric: 62 / 100 (62.0): 100%|██████████| 100/100 [00:00<00:00, 351.07it/s]\n"+ "Average Metric: 257 / 500 (51.4): 100%|██████████| 500/500 [00:01<00:00, 342.65it/s]\n"]},{"name": "stdout","output_type": "stream","text": [- "186\n"+ "Writing dataset with length 771 to trainset_data.jsonl\n"]},{"name": "stderr","output_type": "stream","text": [- "Average Metric: 56 / 100 (56.0): 100%|██████████| 100/100 [00:00<00:00, 343.97it/s]\n"+ "Average Metric: 76 / 125 (60.8): 100%|██████████| 125/125 [00:43<00:00, 2.87it/s]"]},{"name": "stdout","output_type": "stream","text": [- "168\n"+ "Writing dataset with length 228 to devset_data.jsonl\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"]}],@@ -237,9 +209,9 @@"from typing import Callable\n","import ujson\n","\n",- "samples = 100\n",+ "samples = 500\n","\n",- "callback = DataCollectionCallback(num_correct=1, max_attempts=1)\n",+ "callback = DataCollectionCallback(num_correct=1, max_attempts=2)\n","\n","dspy.settings.configure(experimental=True)\n","dc_kwargs = {\n",@@ -249,94 +221,50 @@" \"next_round_dataset_callback\": callback.move_on_callback_correct_with_max,\n"," \"num_threads\": NUM_THREADS,\n","}\n",- "basicmh_bs = BasicMH(**program_params)\n",- "basicmh_bs.load(\"basicmh_3_3_6.json\")\n",- "# NOTE: It looks like all data, not just success is saved\n","\n",- "trainset_data = bootstrap_data_multiple_rounds(basicmh_bs, trainset[:samples], metric, num_rounds=1, **dc_kwargs)\n",- "trainset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in trainset_data if result[\"score\"]], [])\n",- "trainset_data = lm.format_data_for_vanilla_finetuning(trainset_data)\n",- "print(len(trainset_data))\n",- "ts_json = ujson.dumps(trainset_data)\n",- "with open(\"trainset_data.jsonl\", \"w\") as f:\n",- " for line in trainset_data:\n",- " f.write(ujson.dumps(line) + \"\\n\")\n",- "\n",- "devset_data = bootstrap_data_multiple_rounds(basicmh_bs, devset[:int(samples)], metric, num_rounds=1, **dc_kwargs)\n",- "devset_data = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in devset_data if result[\"score\"]], [])\n",- "devset_data = lm.format_data_for_vanilla_finetuning(devset_data)\n",- "print(len(devset_data))\n",- "ds_json = ujson.dumps(devset_data)\n",- "with open(\"devset_data.jsonl\", \"w\") as f:\n",- " for line in devset_data:\n",- " f.write(ujson.dumps(line) + \"\\n\")"+ "dataset_filenames = {\"trainset_data.jsonl\": trainset[:samples], \"devset_data.jsonl\": devset[:int(samples/4)]}\n",+ "\n",+ "def write_data(data, filename):\n",+ " # get the bootstrapped data for num_rounds=1, but using the callback\n",+ " dataset = bootstrap_data_multiple_rounds(basicmh_bs, data, metric, num_rounds=1, **dc_kwargs)\n",+ " # Flatmap the lists of prompt completions gives by each trace\n",+ " dataset = sum([build_prompt_completion_data_from_trace(result[\"trace\"], program=basicmh_bs, exclude_demos=True) for result in dataset if result[\"score\"]], [])\n",+ " # Format the data for finetuning using the LM\n",+ " dataset = lm.format_data_for_vanilla_finetuning(dataset)\n",+ " print(\"Writing dataset with length\", len(dataset), \"to\", filename)\n",+ " with open(filename, \"w\") as f:\n",+ " for line in dataset:\n",+ " f.write(ujson.dumps(line) + \"\\n\")\n",+ "\n",+ "for key, data in dataset_filenames.items():\n",+ " write_data(data, key)"]},{"cell_type": "code",- "execution_count": 11,+ "execution_count": 12,"metadata": {},"outputs": [{- "ename": "APIConnectionError",- "evalue": "Connection error.",+ "ename": "AttributeError",+ "evalue": "'_asyncio.Future' object has no attribute '_condition'","output_type": "error","traceback": ["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mConnectError\u001b[0m Traceback (most recent call last)",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:69\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[0;34m()\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 69\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:233\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 233\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mIterable)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py:216\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_connections(closing)\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py:196\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 195\u001b[0m \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[0;32m--> 196\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_request\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[1;32m 200\u001b[0m \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[1;32m 201\u001b[0m \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[1;32m 202\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:99\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connect_failed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 99\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection\u001b[38;5;241m.\u001b[39mhandle_request(request)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:76\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 76\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m ssl_object \u001b[38;5;241m=\u001b[39m stream\u001b[38;5;241m.\u001b[39mget_extra_info(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mssl_object\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_sync/connection.py:122\u001b[0m, in \u001b[0;36mHTTPConnection._connect\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconnect_tcp\u001b[39m\u001b[38;5;124m\"\u001b[39m, logger, request, kwargs) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[0;32m--> 122\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_network_backend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect_tcp\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 123\u001b[0m trace\u001b[38;5;241m.\u001b[39mreturn_value \u001b[38;5;241m=\u001b[39m stream\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_backends/sync.py:213\u001b[0m, in \u001b[0;36mSyncBackend.connect_tcp\u001b[0;34m(self, host, port, timeout, local_address, socket_options)\u001b[0m\n\u001b[1;32m 212\u001b[0m sock\u001b[38;5;241m.\u001b[39msetsockopt(\u001b[38;5;241m*\u001b[39moption) \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m sock\u001b[38;5;241m.\u001b[39msetsockopt(socket\u001b[38;5;241m.\u001b[39mIPPROTO_TCP, socket\u001b[38;5;241m.\u001b[39mTCP_NODELAY, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SyncStream(sock)\n",- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py:135\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgen\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mthrow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraceback\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;66;03m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[38;5;66;03m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpcore/_exceptions.py:14\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[0;34m(map)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(exc, from_exc):\n\u001b[0;32m---> 14\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m to_exc(exc) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n",- "\u001b[0;31mConnectError\u001b[0m: [Errno 8] nodename nor servname provided, or not known",- "\nThe above exception was the direct cause of the following exception:\n",- "\u001b[0;31mConnectError\u001b[0m Traceback (most recent call last)",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:978\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 977\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 978\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 979\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 980\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_should_stream_response_body\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 981\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 982\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 983\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mTimeoutException \u001b[38;5;28;01mas\u001b[39;00m err:\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:914\u001b[0m, in \u001b[0;36mClient.send\u001b[0;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[1;32m 912\u001b[0m auth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_request_auth(request, auth)\n\u001b[0;32m--> 914\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_auth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 915\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 916\u001b[0m \u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 917\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 918\u001b[0m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 919\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 920\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:942\u001b[0m, in \u001b[0;36mClient._send_handling_auth\u001b[0;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 942\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_redirects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 943\u001b[0m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 945\u001b[0m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhistory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 946\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 947\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:979\u001b[0m, in \u001b[0;36mClient._send_handling_redirects\u001b[0;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[1;32m 977\u001b[0m hook(request)\n\u001b[0;32m--> 979\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_single_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 980\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_client.py:1015\u001b[0m, in \u001b[0;36mClient._send_single_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 1014\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request\u001b[38;5;241m=\u001b[39mrequest):\n\u001b[0;32m-> 1015\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mtransport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1017\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, SyncByteStream)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:233\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 233\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool\u001b[38;5;241m.\u001b[39mhandle_request(req)\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mIterable)\n",- "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py:135\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgen\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mthrow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraceback\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;66;03m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[38;5;66;03m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/httpx/_transports/default.py:86\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[0;34m()\u001b[0m\n\u001b[1;32m 85\u001b[0m message \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(exc)\n\u001b[0;32m---> 86\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m mapped_exc(message) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",- "\u001b[0;31mConnectError\u001b[0m: [Errno 8] nodename nor servname provided, or not known",- "\nThe above exception was the direct cause of the following exception:\n",- "\u001b[0;31mAPIConnectionError\u001b[0m Traceback (most recent call last)",- "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mlm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart_training\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtrainset_data.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdevset_data.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSFT\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:339\u001b[0m, in \u001b[0;36mOpenAIModel.start_training\u001b[0;34m(self, train_path, val_path, method, **kwargs)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOnly SFT finetuning is supported at the moment.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 338\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstart_remote_training(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m--> 339\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_training\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:345\u001b[0m, in \u001b[0;36mOpenAIModel.wait_for_training\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait_for_training\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 344\u001b[0m \u001b[38;5;66;03m# Wait for the future to complete\u001b[39;00m\n\u001b[0;32m--> 345\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_training_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 346\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m60\u001b[39m)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/dilara-dspy/dsp/modules/openai.py:313\u001b[0m, in \u001b[0;36mOpenAIModel.check_training_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_training_status\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfine_tuning_job_id \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou must start training before checking status\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 313\u001b[0m temp_job \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfine_tuning\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjobs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfine_tuning_job_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m temp_job\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msucceeded\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 315\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/resources/fine_tuning/jobs/jobs.py:180\u001b[0m, in \u001b[0;36mJobs.retrieve\u001b[0;34m(self, fine_tuning_job_id, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fine_tuning_job_id:\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected a non-empty value for `fine_tuning_job_id` but received \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfine_tuning_job_id\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 180\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/fine_tuning/jobs/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mfine_tuning_job_id\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 182\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmake_request_options\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 183\u001b[0m \u001b[43m \u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_headers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_body\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\n\u001b[1;32m 184\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 185\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mFineTuningJob\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1209\u001b[0m, in \u001b[0;36mSyncAPIClient.get\u001b[0;34m(self, path, cast_to, options, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1206\u001b[0m opts \u001b[38;5;241m=\u001b[39m FinalRequestOptions\u001b[38;5;241m.\u001b[39mconstruct(method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mget\u001b[39m\u001b[38;5;124m\"\u001b[39m, url\u001b[38;5;241m=\u001b[39mpath, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions)\n\u001b[1;32m 1207\u001b[0m \u001b[38;5;66;03m# cast is required because mypy complains about returning Any even though\u001b[39;00m\n\u001b[1;32m 1208\u001b[0m \u001b[38;5;66;03m# it understands the type variables\u001b[39;00m\n\u001b[0;32m-> 1209\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m cast(ResponseT, \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m)\u001b[49m)\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:942\u001b[0m, in \u001b[0;36mSyncAPIClient.request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 933\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m 934\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 935\u001b[0m cast_to: Type[ResponseT],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 940\u001b[0m stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 941\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[0;32m--> 942\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 943\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 945\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 946\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 947\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 948\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1002\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 999\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered Exception\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 1001\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retries \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1004\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1005\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1006\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1007\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1008\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1009\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1079\u001b[0m, in \u001b[0;36mSyncAPIClient._retry_request\u001b[0;34m(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;66;03m# In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\u001b[39;00m\n\u001b[1;32m 1076\u001b[0m \u001b[38;5;66;03m# different thread if necessary.\u001b[39;00m\n\u001b[1;32m 1077\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(timeout)\n\u001b[0;32m-> 1079\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1080\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1081\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1082\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1083\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1084\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1085\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1002\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 999\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered Exception\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 1001\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retries \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1004\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1005\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1006\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1007\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1008\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1009\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1079\u001b[0m, in \u001b[0;36mSyncAPIClient._retry_request\u001b[0;34m(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;66;03m# In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\u001b[39;00m\n\u001b[1;32m 1076\u001b[0m \u001b[38;5;66;03m# different thread if necessary.\u001b[39;00m\n\u001b[1;32m 1077\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(timeout)\n\u001b[0;32m-> 1079\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1080\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1081\u001b[0m \u001b[43m \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1082\u001b[0m \u001b[43m \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1083\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1084\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1085\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",- "File \u001b[0;32m~/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/openai/_base_client.py:1012\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1002\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_retry_request(\n\u001b[1;32m 1003\u001b[0m input_options,\n\u001b[1;32m 1004\u001b[0m cast_to,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1008\u001b[0m response_headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1009\u001b[0m )\n\u001b[1;32m 1011\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRaising connection error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1012\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m APIConnectionError(request\u001b[38;5;241m=\u001b[39mrequest) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m 1014\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\n\u001b[1;32m 1015\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mHTTP Response: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%i\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 1016\u001b[0m request\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1020\u001b[0m response\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1022\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest_id: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, response\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mx-request-id\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n",- "\u001b[0;31mAPIConnectionError\u001b[0m: Connection error."+ "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",+ "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m future_lm \u001b[38;5;241m=\u001b[39m lm\u001b[38;5;241m.\u001b[39mget_finetune(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrainset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdevset_data.jsonl\u001b[39m\u001b[38;5;124m\"\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSFT\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# finetuned_lm = future_lm.result()\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m finetuned_lm \u001b[38;5;241m=\u001b[39m \u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfuture_lm\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_when\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfutures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIRST_COMPLETED\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mresult() \n\u001b[1;32m 6\u001b[0m finetuned_lm\u001b[38;5;241m.\u001b[39mkwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n",+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:289\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(fs, timeout, return_when)\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait\u001b[39m(fs, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, return_when\u001b[38;5;241m=\u001b[39mALL_COMPLETED):\n\u001b[1;32m 266\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Wait for the futures in the given sequence to complete.\u001b[39;00m\n\u001b[1;32m 267\u001b[0m \n\u001b[1;32m 268\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;124;03m futures.\u001b[39;00m\n\u001b[1;32m 288\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _AcquireFutures(fs):\n\u001b[1;32m 290\u001b[0m done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m fs\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED_AND_NOTIFIED, FINISHED])\n\u001b[1;32m 292\u001b[0m not_done \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(fs) \u001b[38;5;241m-\u001b[39m done\n",+ "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:151\u001b[0m, in \u001b[0;36m_AcquireFutures.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__enter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 150\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfutures:\n\u001b[0;32m--> 151\u001b[0m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241m.\u001b[39macquire()\n",+ "\u001b[0;31mAttributeError\u001b[0m: '_asyncio.Future' object has no attribute '_condition'"]}],"source": [- "lm.start_training(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",- "# ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\n",- "# teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"+ "import concurrent.futures\n",+ "\n",+ "future_lm = lm.get_finetune(\"trainset_data.jsonl\", \"devset_data.jsonl\", method=\"SFT\")\n",+ "finetuned_lm = await future_lm\n",+ "finetuned_lm.kwargs[\"temperature\"] = 0.0"]},{@@ -345,25 +273,7 @@"metadata": {},"outputs": [],"source": [- "TRAIN_FROM_SCRATCH = False\n",- "\n",- "if TRAIN_FROM_SCRATCH:\n",- " teacher = basicmh_bs\n",- " \n",- " # something like\n",- " # Go through all the possible methods + save/load\n",- " # change to future[lm] first\n",- "\n",- "\n",- " # TODO: Working here\n",- " # teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)\n",- "# kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",- "# # NOTE: LM IS NOT SET PROPERLY\n",- "# basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",- "# basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",- "# else:\n",- "# basicmh_bs_ft = BasicMH(**program_params)\n",- "# basicmh_bs_ft.load('mini_ft_hpqa_100.json')"+ "assert finetuned_lm.kwargs[\"model\"] != lm.kwargs[\"model\"]"]},{@@ -372,13 +282,8 @@"metadata": {},"outputs": [],"source": [- "lm2 = lm.copy()\n",- "lm2.kwargs[\"model\"] = \"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\"\n",- "\n","basicmh_bs_ft = BasicMH(**program_params)\n",- "basicmh_bs_ft._set_all_predictor_lms(lm2)\n",- "# print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",- "# dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"+ "basicmh_bs_ft._set_all_predictor_lms(finetuned_lm)"]},{@@ -387,7 +292,7 @@"metadata": {},"outputs": [],"source": [- "RECOMPILE_FT_MODEL = False\n",+ "RECOMPILE_FT_MODEL = True\n","\n","if RECOMPILE_FT_MODEL:\n"," max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",@@ -395,9 +300,11 @@" bsfsrs_teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n"," basicmh_bs_ft_bs = bsfsrs_teleprompter.compile(student=basicmh_bs_ft, trainset=trainset[:100], valset=devset[:250])\n"," basicmh_bs_ft_bs.save('mini_bs_ft_bs_hpqa_100.json')\n",+ " basicmh_bs_ft_bs._set_all_predictor_lms(finetuned_lm)\n","else:\n"," basicmh_bs_ft_bs = BasicMH(**program_params)\n",- " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')"+ " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')\n",+ " basicmh_bs_ft_bs._set_all_predictor_lms(finetuned_lm)"]},{@@ -407,13 +314,14 @@"outputs": [],"source": ["TEST_SIZE = 300\n",- "\n",- "with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",- " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",- " bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",- "with dspy.context(lm=dspy.OpenAI(model=\"ft:gpt-4o-mini-2024-07-18:anyscale:nosys:9vUm0BYv\", max_tokens=500, temperature=0.0)):\n",- " bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",- " bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",+ "baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",+ "baseline_eval._get_lm_info_str()\n",+ "bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",+ "bs_eval._get_lm_info_str()\n",+ "bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",+ "bs_ft_eval._get_lm_info_str()\n",+ "bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",+ "bs_ft_bs_eval._get_lm_info_str()\n","\n","print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to 1 attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n","print(f\"Non-finetuned model: {baseline_eval}\")\n",