Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add OpenAI fine-tune methods

dilarab408892fabbf
diff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.py
index 7fa5b95af..0acaa4bb4 100644
--- a/dspy/clients/finetune.py
+++ b/dspy/clients/finetune.py
@@ -1,11 +1,12 @@
-from abc import ABC, abstractmethod
-from concurrent.futures import ThreadPoolExecutor, Future
+from abc import abstractmethod
+from concurrent.futures import Future
from enum import Enum
import os
-from pathlib import Path
from typing import List, Dict, Any, Optional
import ujson
+from datasets.fingerprint import Hasher
+
from dspy import logger
@@ -13,7 +14,7 @@ from dspy import logger
# Set the directory to save the fine-tuned models
def get_finetune_directory() -> str:
"""Get the directory to save the fine-tuned models."""
- alternative_path = os.path.join(Path.home(), '.dspy_finetune')
+ alternative_path = os.path.join(os.getcwd(), '.dspy_finetune')
# TODO: Should the parent directory be different than those used for
# inference
return os.environ.get('DSPY_FINETUNEDIR') or alternative_path
@@ -33,6 +34,16 @@ class TrainingMethod(str, Enum):
Preference = "Preference"
+class TrainingStatus(str, Enum):
+ """Enum class for remote training status."""
+ not_started = "not_started"
+ pending = "pending"
+ running = "running"
+ succeeded = "succeeded"
+ failed = "failed"
+ cancelled = "cancelled"
+
+
"""Dictionary mapping training methods to the data keys they require."""
TRAINING_METHOD_TO_DATA_KEYS = {
TrainingMethod.SFT: ["prompt", "completion"],
@@ -77,27 +88,40 @@ class FinetuneJob(Future):
raise NotImplementedError("Method `status` is not implemented.")
-def validate_training_data(
+def validate_finetune_data(
data: List[Dict[str, Any]],
- training_method: TrainingMethod
+ train_method: TrainingMethod
) -> Optional[AssertionError]:
- """Validate the training data based on the training method."""
+ """Validate the finetune data based on the training method."""
# Get the required data keys for the training method
- required_keys = TRAINING_METHOD_TO_DATA_KEYS[training_method]
+ required_keys = TRAINING_METHOD_TO_DATA_KEYS[train_method]
# Check if the training data has the required keys
for ind, data_dict in enumerate(data):
- err_msg = f"The datapoint at index {ind} is missing the keys required for {training_method} training."
+ err_msg = f"The datapoint at index {ind} is missing the keys required for {train_method} training."
err_msg = f"\n Expected: {required_keys}"
err_msg = f"\n Found: {data_dict.keys()}"
assert all([key in data_dict for key in required_keys]), err_msg
-def save_data(data: Dict[str, str]) -> str:
- # TODO: Assign different names based on the data hash
+def save_data(
+ data: Any,
+ provider: Optional[str]=None,
+ ) -> str:
+ """Save the fine-tuning data to a file."""
+ # Construct the file name based on the data hash
+ hash = Hasher.hash(data)
+ file_name = f"{hash}.jsonl"
+ file_name = f"{provider}_{file_name}" if provider else file_name
+
+ # Find the directory to save the fine-tuning data
finetune_parent_dir = get_finetune_directory()
- data_path = os.path.join(finetune_parent_dir, "openai", "train.json")
- with open(data_path, "w") as f:
+ os.makedirs(finetune_parent_dir, exist_ok=True)
+
+ # Save the data to a file
+ file_path = os.path.join(finetune_parent_dir, file_name)
+ file_path = os.path.abspath(file_path)
+ with open(file_path, "w") as f:
for item in data:
f.write(ujson.dumps(item) + "\n")
- return data_path
+ return file_path
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 5175fe2be..452ebd3ed 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -241,9 +241,6 @@ def _get_supported_finetune_provider(model: str) -> Union[str, ValueError]:
named `_get_provider` because it does not attempt to find the provider of a
model if there is no DSPy fine-tuning support for it.
"""
- if is_self_hosted_model(model):
- return _PROVIDER_SELF_HOSTED
-
if is_openai_model(model):
return _PROVIDER_OPENAI
@@ -292,18 +289,18 @@ def execute_finetune_job(
# Execute finetune job
job_kwargs = job.get_kwargs()
- if cache_finetune:
- try:
+ logger.error(f"Line 292")
+ try:
+ if cache_finetune:
model = cached_finetune(job=job, **job_kwargs)
- except ValueError as err:
- raise err
- else:
- model = finetune(job=job, **job_kwargs)
-
- # Launch the LM
- lm = LM(model=model, **launch_kwargs)
-
- # Set the result of the finetuning job to the fine-tuned LM
+ else:
+ model = finetune(job=job, **job_kwargs)
+ lm = LM(model=model, **launch_kwargs)
+ except Exception as err:
+ logger.error(err)
+ job.set_result(err)
+
+ # Set the result to the launched lm
job.set_result(lm)
diff --git a/dspy/clients/openai.py b/dspy/clients/openai.py
index d17d0efc7..5a54604e6 100644
--- a/dspy/clients/openai.py
+++ b/dspy/clients/openai.py
@@ -1,28 +1,29 @@
+from collections import defaultdict
import re
import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
+
+import openai
from dspy import logger
from dspy.clients.finetune import (
FinetuneJob,
TrainingMethod,
- validate_training_data,
+ TrainingStatus,
+ validate_finetune_data,
save_data,
)
#-------------------------------------------------------------------------------
-# Variables
+# Function and variables to check if a model is an OpenAI model
#-------------------------------------------------------------------------------
-# List of training methods supported by OpenAI
-TRAINING_METHODS_OPENAI = [
- TrainingMethod.SFT,
-]
+# Provider name
+PROVIDER_OPENAI = "openai"
-# TODO: Should we include the TTS/Dall-E models on this list?
-# List of OpenAI model IDs
-OPENAI_MODEL_IDS = model_ids = [
+# List of model IDs
+_MODEL_IDS = [
"gpt-4o",
"gpt-4o-2024-08-06",
"gpt-4o-2024-05-13",
@@ -64,36 +65,15 @@ OPENAI_MODEL_IDS = model_ids = [
]
-#-------------------------------------------------------------------------------
-# Function and classes required for the fine-tune interface
-#-------------------------------------------------------------------------------
-
-class FinetuneJobOpenAI(FinetuneJob):
-
- def cancel(self):
- """Cancel the finetune job."""
- logger.info("[Finetune] Canceling the OpenAI finetune job")
- time.sleep(3)
- logger.info("[Finetune] Done")
- super().cancel()
-
- def status(self):
- """Get the status of the finetune job."""
- logger.info("[Finetune] Getting the status of the OpenAI finetune job")
- time.sleep(3)
- status = "Running"
- logger.info("[Finetune] Done")
- return status
-
-
def is_openai_model(model: str) -> bool:
"""Check if the model is an OpenAI model."""
- # Filter the "openai/" prefix, if exists
- if model.startswith("openai/"):
- model = model[len("openai/"):]
+ # Filter the provider_prefix, if exists
+ provider_prefix = f"{PROVIDER_OPENAI}/"
+ if model.startswith(provider_prefix):
+ model = model[len(provider_prefix):]
# Check if the model is a base OpenAI model
- if model in OPENAI_MODEL_IDS:
+ if model in _MODEL_IDS:
return True
# Check if the model is a fine-tuned OpneAI model. Fine-tuned OpenAI models
@@ -104,7 +84,7 @@ def is_openai_model(model: str) -> bool:
# by making a call to the OpenAI API to be more exact, but this might
# require an API key with the right permissions.
match = re.match(r"ft:([^:]+):", model)
- if match and match.group(1) in OPENAI_MODEL_IDS:
+ if match and match.group(1) in _MODEL_IDS:
return True
# If the model is not a base OpenAI model or a fine-tuned OpenAI model, then
@@ -112,62 +92,276 @@ def is_openai_model(model: str) -> bool:
return False
+#-------------------------------------------------------------------------------
+# Function and classes required for the fine-tune interface
+#-------------------------------------------------------------------------------
+
+class FinetuneJobOpenAI(FinetuneJob):
+
+ def __init__(self, *args, **kwargs):
+ self.provider_file_id = None # TODO: Can we get this using the job_id?
+ self.provider_job_id = None
+ super().__init__(*args, **kwargs)
+
+ def cancel(self):
+ # Cancel the provider job
+ if _does_job_exist(self.provider_job_id):
+ status = _get_training_status(self.provider_job_id)
+ if _is_terminal_training_status(status):
+ err_msg = "Jobs that are complete cannot be canceled."
+ err_msg += f" Job with ID {self.provider_job_id} is done."
+ raise Exception(err_msg)
+ openai.fine_tuning.jobs.cancel(self.provider_job_id)
+ self.provider_job_id = None
+
+ # Delete the provider file
+ # TODO: Should there be a separate clean method?
+ if self.provider_file_id is not None:
+ if _does_file_exist(self.provider_file_id):
+ openai.files.delete(self.provider_file_id)
+ self.provider_file_id = None
+
+ # Call the super's cancel method after the custom cancellation logic
+ super().cancel()
+
+ def status(self) -> TrainingStatus:
+ status = _get_training_status(self.provider_job_id)
+ return status
+
+
def finetune_openai(
job: FinetuneJobOpenAI,
model: str,
message_completion_pairs: List[Dict[str, str]],
train_kwargs: Optional[Dict[str, Any]]=None,
- ) -> str:
- """Fine-tune an OpenAI model."""
- # Fake fine-tuning
+ ) -> Union[str, Exception]:
train_kwargs = train_kwargs or {}
+ train_data = message_completion_pairs
+ train_method = TrainingMethod.SFT # Note: This could be an argument
+ # Validate train data and method
+ logger.info("[Finetune] Validating the formatting of the data")
+ _validate_data(train_data, train_method)
+ logger.info("[Finetune] Done!")
+
+ # Convert to the OpenAI format
+ logger.info("[Finetune] Converting the data to the OpenAI format")
+ # TODO: Should we use the system prompt?
+ train_data = _convert_data(train_data)
+ logger.info("[Finetune] Done!")
+
+ # Save to a file
+ logger.info("[Finetune] Saving the data to a file")
+ data_path = save_data(train_data, provider=PROVIDER_OPENAI)
+ logger.info("[Finetune] Done!")
+
+ # Upload the data to the cloud
+ logger.info("[Finetune] Uploading the data to the provider")
+ provider_file_id = _upload_data(data_path)
+ job.provider_file_id = provider_file_id
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Start remote training")
+ # We utilize model and train_kwargs here
+ provider_job_id = _start_remote_training(
+ train_file_id=job.provider_file_id,
+ model=model,
+ train_kwargs=train_kwargs,
+ )
+ job.provider_job_id = provider_job_id
+ # job.provider_job_id = "ftjob-ZdEL1mUDk0dwdDuZJQOng8Vv"
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Wait for training to complete")
+ # TODO: Would it be possible to stream the logs?
+ _wait_for_job(job)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Get trained model if the run was a success")
+ model = _get_trained_model(job)
+ logger.info("[Finetune] Done!")
+
+ return model
+
+
+#-------------------------------------------------------------------------------
+# Custom functions to support the finetune_* function
+#-------------------------------------------------------------------------------
+
+_SUPPORTED_TRAINING_METHODS = [
+ TrainingMethod.SFT,
+]
+
+def _get_training_status(job_id: str) -> Union[TrainingStatus, Exception]:
+ # TODO: Should this type be shared across all fine-tune clients?
+ provider_status_to_training_status = {
+ "validating_files": TrainingStatus.pending,
+ "queued": TrainingStatus.pending,
+ "running": TrainingStatus.running,
+ "succeeded": TrainingStatus.succeeded,
+ "failed": TrainingStatus.failed,
+ "cancelled": TrainingStatus.cancelled,
+ }
+
+ # Check if there is an active job
+ if job_id is None:
+ logger.info("There is no active job.")
+ return TrainingStatus.not_started
+
+ err_msg = f"Job with ID {job_id} does not exist."
+ assert _does_job_exist(job_id), err_msg
+
+ # Retrieve the provider's job and report the status
+ provider_job = openai.fine_tuning.jobs.retrieve(job_id)
+ provider_status = provider_job.status
+ status = provider_status_to_training_status[provider_status]
+
+ return status
+
+
+def _does_job_exist(job_id: str) -> bool:
+ try:
+ # TODO: Error handling is vague
+ openai.fine_tuning.jobs.retrieve(job_id)
+ return True
+ except Exception as e:
+ return False
+
+
+def _does_file_exist(file_id: str) -> bool:
try:
- # Validate the formatting of the fine-tuning data
- logger.info("[Finetune] Validating the formatting of the data")
- _validate_data(message_completion_pairs)
- logger.info("[Finetune] Done!")
-
- # Validate the formatting of the fine-tuning data
- logger.error("[Finetune] Saving the data to a file")
- # data_path = save_data(message_completion_pairs)
- time.sleep(1)
- logger.info("[Finetune] Done!")
+ # TODO: Error handling is vague
+ openai.files.retrieve(file_id)
+ return True
+ except Exception as e:
+ return False
+
+
+def _is_terminal_training_status(status: TrainingStatus) -> bool:
+ return status in [
+ TrainingStatus.succeeded,
+ TrainingStatus.failed,
+ TrainingStatus.cancelled,
+ ]
+
+
+def _validate_data(
+ data: Dict[str, str],
+ train_method: TrainingMethod
+ ) -> Optional[Exception]:
+ # Check if this train method is supported
+ if train_method not in _SUPPORTED_TRAINING_METHODS:
+ err_msg = f"OpenAI does not support the training method {train_method}."
+ raise ValueError(err_msg)
+
+ validate_finetune_data(data, train_method)
+
+
+def _convert_data(
+ data: List[Dict[str, str]],
+ system_prompt: Optional[str]=None,
+ ) -> Union[List[Dict[str, Any]], Exception]:
+ # Item-wise conversion function
+ def _row_converter(d):
+ messages = [
+ {"role": "user", "content": d["prompt"]},
+ {"role": "assistant", "content": d["completion"]}
+ ]
+ if system_prompt:
+ messages.insert(0, {"role": "system", "content": system_prompt})
+ messages_dict = {"messages": messages}
+ return messages_dict
- logger.error("[Finetune] Uploading the data to the cloud")
- time.sleep(1)
- logger.error("[Finetune] Done!")
+ # Convert the data to the OpenAI format; validate the converted data
+ converted_data = list(map(_row_converter, data))
+ openai_data_validation(converted_data)
+ return converted_data
- logger.error("[Finetune] Launch training")
- # We utilize model and train_kwargs here
- time.sleep(1)
- logger.error("[Finetune] Done!")
- logger.error("[Finetune] Wait for training to complete")
- time.sleep(1)
- logger.error("[Finetune] Done!")
+def _upload_data(data_path: str) -> str:
+ # Upload the data to the provider
+ provider_file = openai.files.create(
+ file=open(data_path, "rb"),
+ purpose="fine-tune",
+ )
+ return provider_file.id
- logger.error("[Finetune] Get trained model client")
- model = "ft:gpt-4o:2024-08-06:THIS_IS_A_REAL_MODEL!!!" # Hardcoded
- time.sleep(1)
- logger.error("[Finetune] Done!")
- logger.error("[Finetune] Exiting finetune_openai")
+def _start_remote_training(
+ train_file_id: str,
+ model: id,
+ train_kwargs: Optional[Dict[str, Any]]=None
+ ) -> str:
+ train_kwargs = train_kwargs or {}
+ provider_job = openai.fine_tuning.jobs.create(
+ model=model,
+ training_file=train_file_id,
+ hyperparameters=train_kwargs,
+ )
+ return provider_job.id
- return model
- except Exception as e:
- logger.error(f"[Finetune] Error: {e}")
- raise e
+def _wait_for_job(
+ job: FinetuneJobOpenAI,
+ poll_frequency: int=60,
+):
+ while not _is_terminal_training_status(job.status()):
+ time.sleep(poll_frequency)
+
+
+def _get_trained_model(job):
+ status = job.status()
+ if status != TrainingStatus.succeeded:
+ err_msg = f"Job status is {status}."
+ err_msg += f" Must be {TrainingStatus.succeeded} to retrieve the model."
+ logger.error(err_msg)
+ raise Exception(err_msg)
+
+ provider_job = openai.fine_tuning.jobs.retrieve(job.provider_job_id)
+ finetuned_model = provider_job.fine_tuned_model
+ return finetuned_model
#-------------------------------------------------------------------------------
-# Custom functions
+# OpenAI utility function(s)
#-------------------------------------------------------------------------------
-def _validate_data(data: Dict[str, str]) -> Optional[Exception]:
- """Validate the formatting of the fine-tuning data."""
- # TODO: Hardcoded train method
- # TODO: Should we skip this validation since the server might have custom
- # requirements?
- training_method = TrainingMethod.SFT
- validate_training_data(data, training_method)
+
+# Adapted from https://cookbook.openai.com/examples/chat_finetuning_data_prep
+def openai_data_validation(dataset: List[dict[str, Any]]):
+ format_errors = defaultdict(int)
+ for ex in dataset:
+ if not isinstance(ex, dict):
+ format_errors["data_type"] += 1
+ continue
+
+ messages = ex.get("messages", None)
+ if not messages:
+ format_errors["missing_messages_list"] += 1
+ continue
+
+ for message in messages:
+ if "role" not in message or "content" not in message:
+ format_errors["message_missing_key"] += 1
+
+ if any(k not in ("role", "content", "name", "function_call", "weight") for k in message):
+ format_errors["message_unrecognized_key"] += 1
+
+ if message.get("role", None) not in ("system", "user", "assistant", "function"):
+ format_errors["unrecognized_role"] += 1
+
+ content = message.get("content", None)
+ function_call = message.get("function_call", None)
+
+ if (not content and not function_call) or not isinstance(content, str):
+ format_errors["missing_content"] += 1
+
+ if not any(message.get("role", None) == "assistant" for message in messages):
+ format_errors["example_missing_assistant_message"] += 1
+
+ # Raise an error if there are any format errors
+ if format_errors:
+ err_msg = "Found errors in the dataset format using the OpenAI API."
+ err_msg += " Here are the number of datapoints for each error type:"
+ for k, v in format_errors.items():
+ err_msg += "\n {k}: {v}"
+ raise ValueError(err_msg)