DSPy: The framework for programming—not prompting—language models
Python
Add launch kwargs to LM.finetune()
dilarac9c992c448aa
diff --git a/dspy/clients/anyscale.py b/dspy/clients/anyscale.pyindex c307695a6..02a399259 100644--- a/dspy/clients/anyscale.py+++ b/dspy/clients/anyscale.py@@ -1,9 +1,9 @@import reimport time-from typing import List, Dict, Any+from typing import Any, Dict, List, Optional+import dspyfrom dspy import logger-from dspy import LMfrom dspy.clients.finetune import (FinetuneJob,TrainingMethod,@@ -14,12 +14,15 @@ from dspy.clients.finetune import (class FinetuneJobAnyScale(FinetuneJob):def cancel(self):- """Cancel the finetuning job."""+ """Cancel the finetune job."""raise NotImplementedError("Method `cancel` is not implemented.")+ # Call the super's cancel method after the custom cancellation logic,+ # so that the future can be cancelled+ # super().cancel()- def get_status(self):- """Return the status of the finetuning job."""- raise NotImplementedError("Method `get_status` is not implemented.")+ def status(self):+ """Get the status of the finetune job."""+ raise NotImplementedError("Method `status` is not implemented.")def is_anyscale_model(model: str) -> bool:@@ -28,20 +31,27 @@ def is_anyscale_model(model: str) -> bool:return False-def finetune_anyscale(job: FinetuneJobAnyScale, model: str, message_completion_pairs: List[Dict[str, str]], config: Dict[str, Any]):- """Fine-tune an AnyScale model."""+def finetune_anyscale(+ job: FinetuneJobAnyScale,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ launch_kwargs: Optional[Dict[str, Any]]=None,+ ) -> FinetuneJob:+ """Fine-tune with AnyScale."""# Fake fine-tuning- logger.info("[Finetune] Fake fine-tuning of an AnyScale model")+ logger.info("[Finetune] Fake fine-tuning")try:- logger.info("[Finetune] Verifying the formatting of the fine-tuning data")- training_method = TrainingMethod.SFT # Hardcodedlogger.info("[Finetune] Validate the formatting of the fine-tuning data")+ training_method = TrainingMethod.SFT # Hardcoded+ validate_training_data(message_completion_pairs, training_method)time.sleep(5)logger.info("[Finetune] Done!")logger.info("[Finetune] Saving the data to a file") # TODO: Get parent path, keep same across all finetuning jobs+ # We utilize message_completion_pairs heretime.sleep(1)logger.info("[Finetune] Done!")@@ -49,17 +59,23 @@ def finetune_anyscale(job: FinetuneJobAnyScale, model: str, message_completion_ptime.sleep(1)logger.info("[Finetune] Done!")+ logger.info("[Finetune] Launch training")+ # We utilize model and train_kwargs here+ time.sleep(1)+ logger.info("[Finetune] Done!")+logger.info("[Finetune] Wait for training to complete")time.sleep(1)logger.info("[Finetune] Done!")logger.info("[Finetune] Get trained model client")+ model = "anyscale_model" # Hardcodedtime.sleep(1)logger.info("[Finetune] Done!")- logger.info("[Finetune] Create a DSPy LM object for the trained model") # Will cause a circular import- model_id = "SOME_LOCAL_LM" # Hardcoded- lm = LM(model_id)+ logger.info("[Finetune] Create a DSPy LM object for the trained model")+ # We utilize launch_kwargs here+ lm = dspy.LM(model=model, **launch_kwargs)time.sleep(1)logger.info("[Finetune] Done!")@@ -70,6 +86,3 @@ def finetune_anyscale(job: FinetuneJobAnyScale, model: str, message_completion_pexcept Exception as e:logger.error(f"[Finetune] Error: {e}")raise e-- print()- time.sleep(20)diff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.pyindex f7283c4fb..f79f92356 100644--- a/dspy/clients/finetune.py+++ b/dspy/clients/finetune.py@@ -1,3 +1,4 @@+from abc import ABC, abstractmethodfrom concurrent.futures import ThreadPoolExecutor, Futurefrom enum import Enumfrom typing import List, Dict, Any, Optional@@ -39,17 +40,36 @@ def validate_training_data(class FinetuneJob(Future):+ def __init__(self,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ launch_kwargs: Optional[Dict[str, Any]]=None,+ ):+ self.model = model+ self.message_completion_pairs = message_completion_pairs+ self.train_kwargs = train_kwargs+ self.launch_kwargs = launch_kwargs+ super().__init__()+def __str__(self):- return f"FinetuningJob(model={self.model}, config={self.config})"+ config = dict(+ model=self.model,+ train_kwargs=self.train_kwargs,+ launch_kwargs=self.launch_kwargs,+ )+ return f"FinetuningJob({config})"def __repr__(self):return str(self)- # Subclasses should implement the following methods+ # Subclasses should override the cancel method to cancel the finetune job;+ # then call the super's cancel method so that the future can be cancelled.def cancel(self):- """Cancel the finetuning job."""- raise NotImplementedError("Method `get_status` is not implemented.")+ """Cancel the finetune job."""+ super().cancel()- def get_status(self):- """Cancel the finetuning job."""- raise NotImplementedError("Method `get_status` is not implemented.")+ @abstractmethod+ def status(self):+ """Get the status of the finetune job."""+ raise NotImplementedError("Method `status` is not implemented.")diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 5e2c77422..8da34cc0d 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -3,7 +3,7 @@ import functoolsimport osfrom pathlib import Pathimport re-from typing import Any, Dict, List+from typing import Any, Dict, List, Optionalimport ujson@@ -87,37 +87,49 @@ class LM:self_hosted_model_kill(self)logger.debug(f"`LM.kill()` is called for the auto-launched model {self.model} -- no action is taken.")- def finetune(self, message_completion_pairs: List[Dict[str, str]], config: Dict[str, Any]) -> FinetuneJob:+ def finetune(self,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ launch_kwargs: Optional[Dict[str, Any]]=None,+ ) -> FinetuneJob:"""Send a request to the provider to launch the model, if supported."""# Fine-tuning is experimental and requires the experimental flagfrom dspy import settings as settingserr = "Fine-tuning is an experimental feature and requires `dspy.settings.experimental = True`."assert settings.experimental, err+ # Pack the configuration into a dictionary+ config = dict(+ model=self.model,+ message_completion_pairs=message_completion_pairs,+ train_kwargs=train_kwargs,+ launch_kwargs=launch_kwargs,+ )+# Find the respective finetuning functions and job classesfinetune_function = None- finetune_job = None+ finetune_job_class = Noneif is_openai_model(self.model):finetune_function = finetune_openai- finetune_job = FinetuneJobOpenAI()+ finetune_job_class = FinetuneJobOpenAIelif is_anyscale_model(self.model):finetune_function = finetune_anyscale- finetune_function = finetune_anyscale- finetune_job = FinetuneJobAnyScale()+ finetune_job_class = FinetuneJobAnyScale# Ensure that the model supports fine-tuning- if not finetune_function or not finetune_job:+ if not finetune_function or not finetune_job_class:err = f"Fine-tuning is not supported for the model {self.model}."raise ValueError(err)+ # Initialize the finetune job+ finetune_job = finetune_job_class(**config)+# Start asyncronous trainingexecutor = ThreadPoolExecutor(max_workers=1)executor.submit(finetune_function,finetune_job,- model=self.model,- message_completion_pairs=message_completion_pairs,- config=config+ **config)executor.shutdown(wait=False)diff --git a/dspy/clients/openai.py b/dspy/clients/openai.pyindex 2812f4e82..420de85e0 100644--- a/dspy/clients/openai.py+++ b/dspy/clients/openai.py@@ -1,9 +1,9 @@import reimport time-from typing import List, Dict, Any+from typing import Any, Dict, List, Optional+import dspyfrom dspy import logger-from dspy import LMfrom dspy.clients.finetune import (FinetuneJob,TrainingMethod,@@ -56,17 +56,17 @@ OPENAI_MODEL_IDS = model_ids = [class FinetuneJobOpenAI(FinetuneJob):-+def cancel(self):- """Cancel the finetuning job."""- logger.info("[Finetune] Canceling the OpenAI finetuning job")+ """Cancel the finetune job."""+ logger.info("[Finetune] Canceling the OpenAI finetune job")time.sleep(3)logger.info("[Finetune] Done")super().cancel()- def get_status(self):- """Get the status of the finetuning job."""- logger.info("[Finetune] Getting the status of the OpenAI finetuning job")+ def status(self):+ """Get the status of the finetune job."""+ logger.info("[Finetune] Getting the status of the OpenAI finetune job")time.sleep(3)status = "Running"logger.info("[Finetune] Done")@@ -99,11 +99,18 @@ def is_openai_model(model: str) -> bool:return False-def finetune_openai(job: FinetuneJobOpenAI, model: str, message_completion_pairs: List[Dict[str, str]], config: Dict[str, Any]):+def finetune_openai(+ job: FinetuneJobOpenAI,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ launch_kwargs: Optional[Dict[str, Any]]=None,+ ) -> FinetuneJob:"""Fine-tune an OpenAI model."""-# Fake fine-tuning- logger.info("[Finetune] Fake fine-tuning of an OpenAI model")+ logger.info("[Finetune] Fake fine-tuning")+ train_kwargs = train_kwargs or {}+ launch_kwargs = launch_kwargs or {}try:logger.info("[Finetune] Validate the formatting of the fine-tuning data")@@ -113,6 +120,7 @@ def finetune_openai(job: FinetuneJobOpenAI, model: str, message_completion_pairslogger.info("[Finetune] Done!")logger.info("[Finetune] Saving the data to a file") # TODO: Get parent path, keep same across all finetuning jobs+ # We utilize message_completion_pairs heretime.sleep(1)logger.info("[Finetune] Done!")@@ -120,17 +128,23 @@ def finetune_openai(job: FinetuneJobOpenAI, model: str, message_completion_pairstime.sleep(1)logger.info("[Finetune] Done!")+ logger.info("[Finetune] Launch training")+ # We utilize model and train_kwargs here+ time.sleep(1)+ logger.info("[Finetune] Done!")+logger.info("[Finetune] Wait for training to complete")time.sleep(1)logger.info("[Finetune] Done!")logger.info("[Finetune] Get trained model client")+ model = "ft:gpt-4o:2024-08-06:THIS_IS_A_REAL_MODEL!!!" # Hardcodedtime.sleep(1)logger.info("[Finetune] Done!")- logger.info("[Finetune] Create a DSPy LM object for the trained model") # Will cause a circular import- model_id = "ft:gpt-4o:2024-08-06:THIS_IS_A_REAL_MODEL!!!" # Hardcoded- lm = "FakeLM" # Create an LM client+ logger.info("[Finetune] Create a DSPy LM object for the trained model")+ # We utilize launch_kwargs here+ lm = dspy.LM(model=model, **launch_kwargs)time.sleep(1)logger.info("[Finetune] Done!")@@ -141,6 +155,3 @@ def finetune_openai(job: FinetuneJobOpenAI, model: str, message_completion_pairsexcept Exception as e:logger.error(f"[Finetune] Error: {e}")raise e-- print()- time.sleep(20)diff --git a/dspy/clients/self_hosted.py b/dspy/clients/self_hosted.pyindex 3bad38df6..603d6b3ee 100644--- a/dspy/clients/self_hosted.py+++ b/dspy/clients/self_hosted.py@@ -1,5 +1,3 @@-from typing import List, Dict, Any-from dspy.clients.openai import is_openai_model