DSPy: The framework for programming—not prompting—language models
Python
Add OpenAI fine-tune methods
dilarab408892fabbf
diff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.pyindex 7fa5b95af..0acaa4bb4 100644--- a/dspy/clients/finetune.py+++ b/dspy/clients/finetune.py@@ -1,11 +1,12 @@-from abc import ABC, abstractmethod-from concurrent.futures import ThreadPoolExecutor, Future+from abc import abstractmethod+from concurrent.futures import Futurefrom enum import Enumimport os-from pathlib import Pathfrom typing import List, Dict, Any, Optionalimport ujson+from datasets.fingerprint import Hasher+from dspy import logger@@ -13,7 +14,7 @@ from dspy import logger# Set the directory to save the fine-tuned modelsdef get_finetune_directory() -> str:"""Get the directory to save the fine-tuned models."""- alternative_path = os.path.join(Path.home(), '.dspy_finetune')+ alternative_path = os.path.join(os.getcwd(), '.dspy_finetune')# TODO: Should the parent directory be different than those used for# inferencereturn os.environ.get('DSPY_FINETUNEDIR') or alternative_path@@ -33,6 +34,16 @@ class TrainingMethod(str, Enum):Preference = "Preference"+class TrainingStatus(str, Enum):+ """Enum class for remote training status."""+ not_started = "not_started"+ pending = "pending"+ running = "running"+ succeeded = "succeeded"+ failed = "failed"+ cancelled = "cancelled"++"""Dictionary mapping training methods to the data keys they require."""TRAINING_METHOD_TO_DATA_KEYS = {TrainingMethod.SFT: ["prompt", "completion"],@@ -77,27 +88,40 @@ class FinetuneJob(Future):raise NotImplementedError("Method `status` is not implemented.")-def validate_training_data(+def validate_finetune_data(data: List[Dict[str, Any]],- training_method: TrainingMethod+ train_method: TrainingMethod) -> Optional[AssertionError]:- """Validate the training data based on the training method."""+ """Validate the finetune data based on the training method."""# Get the required data keys for the training method- required_keys = TRAINING_METHOD_TO_DATA_KEYS[training_method]+ required_keys = TRAINING_METHOD_TO_DATA_KEYS[train_method]# Check if the training data has the required keysfor ind, data_dict in enumerate(data):- err_msg = f"The datapoint at index {ind} is missing the keys required for {training_method} training."+ err_msg = f"The datapoint at index {ind} is missing the keys required for {train_method} training."err_msg = f"\n Expected: {required_keys}"err_msg = f"\n Found: {data_dict.keys()}"assert all([key in data_dict for key in required_keys]), err_msg-def save_data(data: Dict[str, str]) -> str:- # TODO: Assign different names based on the data hash+def save_data(+ data: Any,+ provider: Optional[str]=None,+ ) -> str:+ """Save the fine-tuning data to a file."""+ # Construct the file name based on the data hash+ hash = Hasher.hash(data)+ file_name = f"{hash}.jsonl"+ file_name = f"{provider}_{file_name}" if provider else file_name++ # Find the directory to save the fine-tuning datafinetune_parent_dir = get_finetune_directory()- data_path = os.path.join(finetune_parent_dir, "openai", "train.json")- with open(data_path, "w") as f:+ os.makedirs(finetune_parent_dir, exist_ok=True)++ # Save the data to a file+ file_path = os.path.join(finetune_parent_dir, file_name)+ file_path = os.path.abspath(file_path)+ with open(file_path, "w") as f:for item in data:f.write(ujson.dumps(item) + "\n")- return data_path+ return file_pathdiff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 5175fe2be..452ebd3ed 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -241,9 +241,6 @@ def _get_supported_finetune_provider(model: str) -> Union[str, ValueError]:named `_get_provider` because it does not attempt to find the provider of amodel if there is no DSPy fine-tuning support for it."""- if is_self_hosted_model(model):- return _PROVIDER_SELF_HOSTED-if is_openai_model(model):return _PROVIDER_OPENAI@@ -292,18 +289,18 @@ def execute_finetune_job(# Execute finetune jobjob_kwargs = job.get_kwargs()- if cache_finetune:- try:+ logger.error(f"Line 292")+ try:+ if cache_finetune:model = cached_finetune(job=job, **job_kwargs)- except ValueError as err:- raise err- else:- model = finetune(job=job, **job_kwargs)-- # Launch the LM- lm = LM(model=model, **launch_kwargs)-- # Set the result of the finetuning job to the fine-tuned LM+ else:+ model = finetune(job=job, **job_kwargs)+ lm = LM(model=model, **launch_kwargs)+ except Exception as err:+ logger.error(err)+ job.set_result(err)++ # Set the result to the launched lmjob.set_result(lm)diff --git a/dspy/clients/openai.py b/dspy/clients/openai.pyindex d17d0efc7..5a54604e6 100644--- a/dspy/clients/openai.py+++ b/dspy/clients/openai.py@@ -1,28 +1,29 @@+from collections import defaultdictimport reimport time-from typing import Any, Dict, List, Optional+from typing import Any, Dict, List, Optional, Union++import openaifrom dspy import loggerfrom dspy.clients.finetune import (FinetuneJob,TrainingMethod,- validate_training_data,+ TrainingStatus,+ validate_finetune_data,save_data,)#--------------------------------------------------------------------------------# Variables+# Function and variables to check if a model is an OpenAI model#--------------------------------------------------------------------------------# List of training methods supported by OpenAI-TRAINING_METHODS_OPENAI = [- TrainingMethod.SFT,-]+# Provider name+PROVIDER_OPENAI = "openai"-# TODO: Should we include the TTS/Dall-E models on this list?-# List of OpenAI model IDs-OPENAI_MODEL_IDS = model_ids = [+# List of model IDs+_MODEL_IDS = ["gpt-4o","gpt-4o-2024-08-06","gpt-4o-2024-05-13",@@ -64,36 +65,15 @@ OPENAI_MODEL_IDS = model_ids = []-#--------------------------------------------------------------------------------# Function and classes required for the fine-tune interface-#---------------------------------------------------------------------------------class FinetuneJobOpenAI(FinetuneJob):-- def cancel(self):- """Cancel the finetune job."""- logger.info("[Finetune] Canceling the OpenAI finetune job")- time.sleep(3)- logger.info("[Finetune] Done")- super().cancel()-- def status(self):- """Get the status of the finetune job."""- logger.info("[Finetune] Getting the status of the OpenAI finetune job")- time.sleep(3)- status = "Running"- logger.info("[Finetune] Done")- return status--def is_openai_model(model: str) -> bool:"""Check if the model is an OpenAI model."""- # Filter the "openai/" prefix, if exists- if model.startswith("openai/"):- model = model[len("openai/"):]+ # Filter the provider_prefix, if exists+ provider_prefix = f"{PROVIDER_OPENAI}/"+ if model.startswith(provider_prefix):+ model = model[len(provider_prefix):]# Check if the model is a base OpenAI model- if model in OPENAI_MODEL_IDS:+ if model in _MODEL_IDS:return True# Check if the model is a fine-tuned OpneAI model. Fine-tuned OpenAI models@@ -104,7 +84,7 @@ def is_openai_model(model: str) -> bool:# by making a call to the OpenAI API to be more exact, but this might# require an API key with the right permissions.match = re.match(r"ft:([^:]+):", model)- if match and match.group(1) in OPENAI_MODEL_IDS:+ if match and match.group(1) in _MODEL_IDS:return True# If the model is not a base OpenAI model or a fine-tuned OpenAI model, then@@ -112,62 +92,276 @@ def is_openai_model(model: str) -> bool:return False+#-------------------------------------------------------------------------------+# Function and classes required for the fine-tune interface+#-------------------------------------------------------------------------------++class FinetuneJobOpenAI(FinetuneJob):++ def __init__(self, *args, **kwargs):+ self.provider_file_id = None # TODO: Can we get this using the job_id?+ self.provider_job_id = None+ super().__init__(*args, **kwargs)++ def cancel(self):+ # Cancel the provider job+ if _does_job_exist(self.provider_job_id):+ status = _get_training_status(self.provider_job_id)+ if _is_terminal_training_status(status):+ err_msg = "Jobs that are complete cannot be canceled."+ err_msg += f" Job with ID {self.provider_job_id} is done."+ raise Exception(err_msg)+ openai.fine_tuning.jobs.cancel(self.provider_job_id)+ self.provider_job_id = None++ # Delete the provider file+ # TODO: Should there be a separate clean method?+ if self.provider_file_id is not None:+ if _does_file_exist(self.provider_file_id):+ openai.files.delete(self.provider_file_id)+ self.provider_file_id = None++ # Call the super's cancel method after the custom cancellation logic+ super().cancel()++ def status(self) -> TrainingStatus:+ status = _get_training_status(self.provider_job_id)+ return status++def finetune_openai(job: FinetuneJobOpenAI,model: str,message_completion_pairs: List[Dict[str, str]],train_kwargs: Optional[Dict[str, Any]]=None,- ) -> str:- """Fine-tune an OpenAI model."""- # Fake fine-tuning+ ) -> Union[str, Exception]:train_kwargs = train_kwargs or {}+ train_data = message_completion_pairs+ train_method = TrainingMethod.SFT # Note: This could be an argument+ # Validate train data and method+ logger.info("[Finetune] Validating the formatting of the data")+ _validate_data(train_data, train_method)+ logger.info("[Finetune] Done!")++ # Convert to the OpenAI format+ logger.info("[Finetune] Converting the data to the OpenAI format")+ # TODO: Should we use the system prompt?+ train_data = _convert_data(train_data)+ logger.info("[Finetune] Done!")++ # Save to a file+ logger.info("[Finetune] Saving the data to a file")+ data_path = save_data(train_data, provider=PROVIDER_OPENAI)+ logger.info("[Finetune] Done!")++ # Upload the data to the cloud+ logger.info("[Finetune] Uploading the data to the provider")+ provider_file_id = _upload_data(data_path)+ job.provider_file_id = provider_file_id+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Start remote training")+ # We utilize model and train_kwargs here+ provider_job_id = _start_remote_training(+ train_file_id=job.provider_file_id,+ model=model,+ train_kwargs=train_kwargs,+ )+ job.provider_job_id = provider_job_id+ # job.provider_job_id = "ftjob-ZdEL1mUDk0dwdDuZJQOng8Vv"+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Wait for training to complete")+ # TODO: Would it be possible to stream the logs?+ _wait_for_job(job)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Get trained model if the run was a success")+ model = _get_trained_model(job)+ logger.info("[Finetune] Done!")++ return model+++#-------------------------------------------------------------------------------+# Custom functions to support the finetune_* function+#-------------------------------------------------------------------------------++_SUPPORTED_TRAINING_METHODS = [+ TrainingMethod.SFT,+]++def _get_training_status(job_id: str) -> Union[TrainingStatus, Exception]:+ # TODO: Should this type be shared across all fine-tune clients?+ provider_status_to_training_status = {+ "validating_files": TrainingStatus.pending,+ "queued": TrainingStatus.pending,+ "running": TrainingStatus.running,+ "succeeded": TrainingStatus.succeeded,+ "failed": TrainingStatus.failed,+ "cancelled": TrainingStatus.cancelled,+ }++ # Check if there is an active job+ if job_id is None:+ logger.info("There is no active job.")+ return TrainingStatus.not_started++ err_msg = f"Job with ID {job_id} does not exist."+ assert _does_job_exist(job_id), err_msg++ # Retrieve the provider's job and report the status+ provider_job = openai.fine_tuning.jobs.retrieve(job_id)+ provider_status = provider_job.status+ status = provider_status_to_training_status[provider_status]++ return status+++def _does_job_exist(job_id: str) -> bool:+ try:+ # TODO: Error handling is vague+ openai.fine_tuning.jobs.retrieve(job_id)+ return True+ except Exception as e:+ return False+++def _does_file_exist(file_id: str) -> bool:try:- # Validate the formatting of the fine-tuning data- logger.info("[Finetune] Validating the formatting of the data")- _validate_data(message_completion_pairs)- logger.info("[Finetune] Done!")-- # Validate the formatting of the fine-tuning data- logger.error("[Finetune] Saving the data to a file")- # data_path = save_data(message_completion_pairs)- time.sleep(1)- logger.info("[Finetune] Done!")+ # TODO: Error handling is vague+ openai.files.retrieve(file_id)+ return True+ except Exception as e:+ return False+++def _is_terminal_training_status(status: TrainingStatus) -> bool:+ return status in [+ TrainingStatus.succeeded,+ TrainingStatus.failed,+ TrainingStatus.cancelled,+ ]+++def _validate_data(+ data: Dict[str, str],+ train_method: TrainingMethod+ ) -> Optional[Exception]:+ # Check if this train method is supported+ if train_method not in _SUPPORTED_TRAINING_METHODS:+ err_msg = f"OpenAI does not support the training method {train_method}."+ raise ValueError(err_msg)++ validate_finetune_data(data, train_method)+++def _convert_data(+ data: List[Dict[str, str]],+ system_prompt: Optional[str]=None,+ ) -> Union[List[Dict[str, Any]], Exception]:+ # Item-wise conversion function+ def _row_converter(d):+ messages = [+ {"role": "user", "content": d["prompt"]},+ {"role": "assistant", "content": d["completion"]}+ ]+ if system_prompt:+ messages.insert(0, {"role": "system", "content": system_prompt})+ messages_dict = {"messages": messages}+ return messages_dict- logger.error("[Finetune] Uploading the data to the cloud")- time.sleep(1)- logger.error("[Finetune] Done!")+ # Convert the data to the OpenAI format; validate the converted data+ converted_data = list(map(_row_converter, data))+ openai_data_validation(converted_data)+ return converted_data- logger.error("[Finetune] Launch training")- # We utilize model and train_kwargs here- time.sleep(1)- logger.error("[Finetune] Done!")- logger.error("[Finetune] Wait for training to complete")- time.sleep(1)- logger.error("[Finetune] Done!")+def _upload_data(data_path: str) -> str:+ # Upload the data to the provider+ provider_file = openai.files.create(+ file=open(data_path, "rb"),+ purpose="fine-tune",+ )+ return provider_file.id- logger.error("[Finetune] Get trained model client")- model = "ft:gpt-4o:2024-08-06:THIS_IS_A_REAL_MODEL!!!" # Hardcoded- time.sleep(1)- logger.error("[Finetune] Done!")- logger.error("[Finetune] Exiting finetune_openai")+def _start_remote_training(+ train_file_id: str,+ model: id,+ train_kwargs: Optional[Dict[str, Any]]=None+ ) -> str:+ train_kwargs = train_kwargs or {}+ provider_job = openai.fine_tuning.jobs.create(+ model=model,+ training_file=train_file_id,+ hyperparameters=train_kwargs,+ )+ return provider_job.id- return model- except Exception as e:- logger.error(f"[Finetune] Error: {e}")- raise e+def _wait_for_job(+ job: FinetuneJobOpenAI,+ poll_frequency: int=60,+):+ while not _is_terminal_training_status(job.status()):+ time.sleep(poll_frequency)+++def _get_trained_model(job):+ status = job.status()+ if status != TrainingStatus.succeeded:+ err_msg = f"Job status is {status}."+ err_msg += f" Must be {TrainingStatus.succeeded} to retrieve the model."+ logger.error(err_msg)+ raise Exception(err_msg)++ provider_job = openai.fine_tuning.jobs.retrieve(job.provider_job_id)+ finetuned_model = provider_job.fine_tuned_model+ return finetuned_model#--------------------------------------------------------------------------------# Custom functions+# OpenAI utility function(s)#--------------------------------------------------------------------------------def _validate_data(data: Dict[str, str]) -> Optional[Exception]:- """Validate the formatting of the fine-tuning data."""- # TODO: Hardcoded train method- # TODO: Should we skip this validation since the server might have custom- # requirements?- training_method = TrainingMethod.SFT- validate_training_data(data, training_method)++# Adapted from https://cookbook.openai.com/examples/chat_finetuning_data_prep+def openai_data_validation(dataset: List[dict[str, Any]]):+ format_errors = defaultdict(int)+ for ex in dataset:+ if not isinstance(ex, dict):+ format_errors["data_type"] += 1+ continue++ messages = ex.get("messages", None)+ if not messages:+ format_errors["missing_messages_list"] += 1+ continue++ for message in messages:+ if "role" not in message or "content" not in message:+ format_errors["message_missing_key"] += 1++ if any(k not in ("role", "content", "name", "function_call", "weight") for k in message):+ format_errors["message_unrecognized_key"] += 1++ if message.get("role", None) not in ("system", "user", "assistant", "function"):+ format_errors["unrecognized_role"] += 1++ content = message.get("content", None)+ function_call = message.get("function_call", None)++ if (not content and not function_call) or not isinstance(content, str):+ format_errors["missing_content"] += 1++ if not any(message.get("role", None) == "assistant" for message in messages):+ format_errors["example_missing_assistant_message"] += 1++ # Raise an error if there are any format errors+ if format_errors:+ err_msg = "Found errors in the dataset format using the OpenAI API."+ err_msg += " Here are the number of datapoints for each error type:"+ for k, v in format_errors.items():+ err_msg += "\n {k}: {v}"+ raise ValueError(err_msg)