DSPy: The framework for programming—not prompting—language models
Python
Merge main
dilarae7d5046eac1e
diff --cc dspy/clients/anyscale.pyindex 43220f6b6,000000000..dea3ca4c5mode 100644,000000..100644--- a/dspy/clients/anyscale.py+++ b/dspy/clients/anyscale.py@@@ -1,109 -1,0 +1,107 @@@- import re+import time+from typing import Any, Dict, List, Optional+- import dspy- from dspy import logger++from dspy.utils.logging import logger+from dspy.clients.finetune import (+ FinetuneJob,+ TrainingMethod,+ get_finetune_directory,+ validate_finetune_data,+)++#-------------------------------------------------------------------------------+# Variables+#-------------------------------------------------------------------------------++# List of training methods supported by AnyScale+TRAINING_METHODS_ANYSCALE = [+ TrainingMethod.SFT,+]++#-------------------------------------------------------------------------------+# Launching and killing LMs+#-------------------------------------------------------------------------------++def anyscale_model_launch(model: str, launch_kwargs: Dict[str, Any]):+ """Launch an AnyScale model."""+ # TODO: Hardcode resources for launching a select server through docker+ raise NotImplementedError("Method `anyscale_model_launch` is not implemented.")+++def anyscale_model_kill(model: str, launch_kwargs: Dict[str, Any]):+ # TODO: Hardcode resources for killing a select server through docker+ """Kill an AnyScale model."""+ raise NotImplementedError("Method `anyscale_model_kill` is not implemented.")+++#-------------------------------------------------------------------------------+# Function and classes required for the fine-tune interface+#-------------------------------------------------------------------------------++class FinetuneJobAnyScale(FinetuneJob):++ def cancel(self):+ """Cancel the finetune job."""+ raise NotImplementedError("Method `cancel` is not implemented.")+ # Call the super's cancel method after the custom cancellation logic,+ # so that the future can be cancelled+ # super().cancel()++ def status(self):+ """Get the status of the finetune job."""+ raise NotImplementedError("Method `status` is not implemented.")+++def is_anyscale_model(model: str) -> bool:+ """Check if the model is an AnyScale model."""+ logger.info("Is AnyScale model is not implemented, returning False as a default to not break lm.py")+ return False+++def finetune_anyscale(+ job: FinetuneJobAnyScale,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ ) -> str:+ """Fine-tune with AnyScale."""+ # Fake fine-tuning+ logger.info("[Finetune] Fake fine-tuning")+ train_kwargs = train_kwargs or {}++ try:+ logger.info("[Finetune] Validate the formatting of the fine-tuning data")+ training_method = TrainingMethod.SFT # Hardcoded+ validate_finetune_data(message_completion_pairs, training_method)+ time.sleep(5)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Saving the data to a file")+ finetune_parent_dir = get_finetune_directory()+ # We utilize message_completion_pairs here+ time.sleep(1)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Uploading the data to the cloud")+ time.sleep(1)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Launch training")+ # We utilize model and train_kwargs here+ time.sleep(1)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Wait for training to complete")+ time.sleep(1)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Get trained model client")+ model = "anyscale_model" # Hardcoded+ time.sleep(1)+ logger.info("[Finetune] Done!")++ return model++ except Exception as e:+ logger.error(f"[Finetune] Error: {e}")+ raise ediff --cc dspy/clients/finetune.pyindex a6e6e0005,000000000..3c14be363mode 100644,000000..100644--- a/dspy/clients/finetune.py+++ b/dspy/clients/finetune.py@@@ -1,125 -1,0 +1,123 @@@+from abc import abstractmethod+from concurrent.futures import Future+from enum import Enum+import os+from typing import List, Dict, Any, Optional+import ujson++from datasets.fingerprint import Hasher+- from dspy import logger-++# TODO: Move to a centralized location with all the other environment variables+# Set the directory to save the fine-tuned models+def get_finetune_directory() -> str:+ """Get the directory to save the fine-tuned models."""+ alternative_path = os.path.join(os.getcwd(), '.dspy_finetune')+ return os.environ.get('DSPY_FINETUNEDIR') or alternative_path+++FINETUNE_DIRECTORY = get_finetune_directory()+++class TrainingMethod(str, Enum):+ """Enum class for training methods.++ When comparing enums, Python checks for object IDs, which means that the+ enums can't be compared directly. Subclassing the Enum class along with the+ str class allows for direct comparison of the enums.+ """+ SFT = "SFT"+ Preference = "Preference"+++class TrainingStatus(str, Enum):+ """Enum class for remote training status."""+ not_started = "not_started"+ pending = "pending"+ running = "running"+ succeeded = "succeeded"+ failed = "failed"+ cancelled = "cancelled"+++"""Dictionary mapping training methods to the data keys they require."""+TRAINING_METHOD_TO_DATA_KEYS = {+ TrainingMethod.SFT: ["prompt", "completion"],+ TrainingMethod.Preference: ["prompt", "chosen", "rejected"],+}+++class FinetuneJob(Future):++ def __init__(self,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ ):+ self.model = model+ self.message_completion_pairs = message_completion_pairs+ self.train_kwargs: Dict[str, Any] = train_kwargs or {}+ super().__init__()++ def get_kwargs(self):+ return dict(+ model=self.model,+ message_completion_pairs=self.message_completion_pairs,+ train_kwargs=self.train_kwargs,+ )++ def __str__(self):+ return f"FinetuningJob({self.to_dict()})"++ def __repr__(self):+ return str(self)++ # Subclasses should override the cancel method to cancel the finetune job;+ # then call the super's cancel method so that the future can be cancelled.+ def cancel(self):+ """Cancel the finetune job."""+ super().cancel()++ @abstractmethod+ def status(self):+ """Get the status of the finetune job."""+ raise NotImplementedError("Method `status` is not implemented.")+++def validate_finetune_data(+ data: List[Dict[str, Any]],+ train_method: TrainingMethod+ ) -> Optional[AssertionError]:+ """Validate the finetune data based on the training method."""+ # Get the required data keys for the training method+ required_keys = TRAINING_METHOD_TO_DATA_KEYS[train_method]++ # Check if the training data has the required keys+ for ind, data_dict in enumerate(data):+ err_msg = f"The datapoint at index {ind} is missing the keys required for {train_method} training."+ err_msg = f"\n Expected: {required_keys}"+ err_msg = f"\n Found: {data_dict.keys()}"+ assert all([key in data_dict for key in required_keys]), err_msg+++def save_data(+ data: List[Dict[str, Any]],+ provider_name: Optional[str]=None,+ ) -> str:+ """Save the fine-tuning data to a file."""+ # Construct the file name based on the data hash+ hash = Hasher.hash(data)+ file_name = f"{hash}.jsonl"+ file_name = f"{provider_name}_{file_name}" if provider_name else file_name++ # Find the directory to save the fine-tuning data+ finetune_parent_dir = get_finetune_directory()+ os.makedirs(finetune_parent_dir, exist_ok=True)++ # Save the data to a file+ file_path = os.path.join(finetune_parent_dir, file_name)+ file_path = os.path.abspath(file_path)+ with open(file_path, "w") as f:+ for item in data:+ f.write(ujson.dumps(item) + "\n")+ return file_pathdiff --cc dspy/clients/lm.pyindex b72fc41f7,994c8bc41..014025efc--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@@ -1,62 -1,23 +1,53 @@@+from concurrent.futures import ThreadPoolExecutor++from datetime import datetimeimport functoolsimport os-import uuid-from datetime import datetimefrom pathlib import Path+from typing import Any, Dict, List, Optional, Type, Union+import ujson++import uuid++- from dspy import logger++from dspy.utils.logging import logger+from dspy.clients.finetune import FinetuneJob+from dspy.clients.self_hosted import (+ is_self_hosted_model,+ self_hosted_model_launch,+ self_hosted_model_kill,+)+from dspy.clients.anyscale import (+ is_anyscale_model,+ anyscale_model_launch,+ anyscale_model_kill,+)+ import litellm-import ujson+ from litellm.caching import Cache- DISK_CACHE_DIR = os.environ.get('DSPY_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache')-disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")-litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")++DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")++litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")+ litellm.telemetry = False-- try:- import warnings- with warnings.catch_warnings():- warnings.filterwarnings("ignore", category=UserWarning)- if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:- os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"- import litellm- litellm.telemetry = False-- from litellm.caching import Cache- litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"- except ImportError:- class LitellmPlaceholder:- def __getattr__(self, _): raise ImportError("The LiteLLM package is not installed. Run `pip install litellm`.")-- litellm = LitellmPlaceholder()++#-------------------------------------------------------------------------------+# LiteLLM Client+#-------------------------------------------------------------------------------+class LM:- def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs):+ def __init__(self,+ model,+ model_type='chat',+ temperature=0.0,+ max_tokens=1000,+ cache=True,+ launch_kwargs=None,+ **kwargs+ ):+ # Remember to update LM.copy() if you modify the constructor!self.model = modelself.model_type = model_typeself.cache = cache@@@ -64,10 -24,10 +55,11 @@@self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)self.history = []+ # TODO: This is error prone!if "o1-" in model:- assert max_tokens >= 5000 and temperature == 1.0, \- "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"+ assert (+ max_tokens >= 5000 and temperature == 1.0+ ), "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"def __call__(self, prompt=None, messages=None, **kwargs):# Build the request.@@@ -87,22 -49,17 +81,29 @@@entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response)entry = dict(**entry, outputs=outputs, usage=dict(response["usage"]))entry = dict(**entry, cost=response.get("_hidden_params", {}).get("response_cost"))+ entry = dict(+ **entry,+ timestamp=datetime.now().isoformat(),+ uuid=str(uuid.uuid4()),+ model=self.model,+ model_type=self.model_type,+ )self.history.append(entry)-+return outputs+ def copy(self, **kwargs):+ """Create a copy of the current LM with updated kwargs."""+ kwargs = {+ "model": self.model,+ "model_type": self.model_type,+ "cache": self.cache,+ "launch_kwargs": self.launch_kwargs,+ **self.kwargs,+ **kwargs,+ }+ return self.__class__(**kwargs)+def inspect_history(self, n: int = 1):_inspect_history(self, n)@@@ -221,136 -133,5 +236,136 @@@ def _inspect_history(lm, n: int = 1)if len(outputs) > 1:choices_text = f" \t (and {len(outputs)-1} other completions)"print(_red(choices_text, end=""))-+print("\n\n\n")+++#-------------------------------------------------------------------------------+# Fine-tune support+#-------------------------------------------------------------------------------+# TODO: This part can be moved to a separate module++from dspy.clients.openai import (+ FinetuneJobOpenAI,+ is_openai_model,+ finetune_openai+)+from dspy.clients.anyscale import (+ FinetuneJobAnyScale,+ is_anyscale_model,+ finetune_anyscale+)+++_PROVIDER_ANYSCALE = "anyscale"+_PROVIDER_OPENAI = "openai"+_SUPPORTED_FINETUNE_PROVIDERS = [+ _PROVIDER_ANYSCALE,+ _PROVIDER_OPENAI,+]+++def _get_supported_finetune_provider(model: str) -> Union[str, ValueError]:+ """Return the finetuning provider for the given model.++ The provider must be in _SUPPORTED_FINETUNE_PROVIDERS. This function is not+ named `_get_provider` because it does not attempt to find the provider of a+ model if there is no DSPy fine-tuning support for it.+ """+ if is_openai_model(model):+ return _PROVIDER_OPENAI++ if is_anyscale_model(model):+ return _PROVIDER_ANYSCALE++ return ValueError(f"DSPy does not have fine-tuning support for {model}")+++def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:+ """Get the FinetuneJob class for the provider."""+ # Mapping from provider to finetune job type+ _PROVIDER_TO_FINETUNE_JOB_CLASS = {+ _PROVIDER_ANYSCALE: FinetuneJobAnyScale,+ _PROVIDER_OPENAI: FinetuneJobOpenAI,+ }++ # Get the FinetuneJob class for the provider+ _CLS = _PROVIDER_TO_FINETUNE_JOB_CLASS[provider]++ return _CLS+++def get_provider_finetune_function(provider: str) -> callable:+ """Return the finetune function for the given model."""+ # Mapping from provider to finetune function+ _PROVIDER_TO_FINETUNE_FUNCTION = {+ _PROVIDER_ANYSCALE: finetune_anyscale,+ _PROVIDER_OPENAI: finetune_openai,+ }++ # Get the finetuning provider+ finetune_function = _PROVIDER_TO_FINETUNE_FUNCTION[provider]++ return finetune_function+++def execute_finetune_job(+ job: FinetuneJob[Type[LM]],+ lm: LM,+ cache_finetune: bool=True+):+ """Execute the finetune job in a blocking manner."""+ try:+ job_kwargs = job.get_kwargs()+ if cache_finetune:+ model = cached_finetune(job=job, **job_kwargs)+ else:+ model = finetune(job=job, **job_kwargs)+ lm = lm.copy(model=model)+ job.set_result(lm)+ except Exception as err:+ logger.error(err)+ job.set_result(err)+++# TODO: Perhaps we shouldn't directly cache the data+# TODO: Add DiskCache, ignore job+def cached_finetune(+ job,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+) -> Union[str, ValueError]:+ return finetune(+ job=job,+ model=model,+ message_completion_pairs=message_completion_pairs,+ train_kwargs=train_kwargs,+ )+++def finetune(+ job,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+) -> Union[str, Exception]:+ """Fine-tune a new model based on the given model."""+ # Get the fine-tuning provider+ try:+ provider = _get_supported_finetune_provider(model)++ # Get the finetune function+ provider_finetune_function = get_provider_finetune_function(provider)++ # Fine-tune a new model based on the given model+ model = provider_finetune_function(+ job=job,+ model=model,+ message_completion_pairs=message_completion_pairs,+ train_kwargs=train_kwargs,+ )+ except Exception as err:+ raise err++ return modeldiff --cc dspy/clients/openai.pyindex 95af1f16f,000000000..f7feb961amode 100644,000000..100644--- a/dspy/clients/openai.py+++ b/dspy/clients/openai.py@@@ -1,367 -1,0 +1,367 @@@+from collections import defaultdict+import re+import time+from typing import Any, Dict, List, Optional, Union++import openai+- from dspy import logger++from dspy.utils.logging import logger+from dspy.clients.finetune import (+ FinetuneJob,+ TrainingMethod,+ TrainingStatus,+ validate_finetune_data,+ save_data,+)+++#-------------------------------------------------------------------------------+# Function and variables to check if a model is an OpenAI model+#-------------------------------------------------------------------------------++# Provider name+PROVIDER_OPENAI = "openai"++# List of model IDs+_MODEL_IDS = [+ "gpt-4o",+ "gpt-4o-2024-08-06",+ "gpt-4o-2024-05-13",+ "chatgpt-4o-latest",+ "gpt-4o-mini",+ "gpt-4o-mini-2024-07-18",+ "gpt-4o-realtime-preview",+ "gpt-4o-realtime-preview-2024-10-01",+ "o1-preview",+ "o1-preview-2024-09-12",+ "o1-mini",+ "o1-mini-2024-09-12",+ "gpt-4-turbo",+ "gpt-4-turbo-2024-04-09",+ "gpt-4-turbo-preview",+ "gpt-4-0125-preview",+ "gpt-4-1106-preview",+ "gpt-4",+ "gpt-4-0613",+ "gpt-4-0314",+ "gpt-3.5-turbo-0125",+ "gpt-3.5-turbo",+ "gpt-3.5-turbo-1106",+ "gpt-3.5-turbo-instruct",+ "dall-e-3",+ "dall-e-2",+ "tts-1",+ "tts-1-hd",+ "text-embedding-3-large",+ "text-embedding-3-small",+ "text-embedding-ada-002",+ "omni-moderation-latest",+ "omni-moderation-2024-09-26",+ "text-moderation-latest",+ "text-moderation-stable",+ "text-moderation-007",+ "babbage-002",+ "davinci-002"+]+++def is_openai_model(model: str) -> bool:+ """Check if the model is an OpenAI model."""+ # Filter the provider_prefix, if exists+ provider_prefix = f"{PROVIDER_OPENAI}/"+ if model.startswith(provider_prefix):+ model = model[len(provider_prefix):]++ # Check if the model is a base OpenAI model+ if model in _MODEL_IDS:+ return True++ # Check if the model is a fine-tuned OpneAI model. Fine-tuned OpenAI models+ # have the prefix "ft:<BASE_MODEL_NAME>:", followed by a string specifying+ # the fine-tuned model. The following RegEx pattern is used to match the+ # base model name.+ # TODO: This part can be updated to match the actual fine-tuned model names+ # by making a call to the OpenAI API to be more exact, but this might+ # require an API key with the right permissions.+ match = re.match(r"ft:([^:]+):", model)+ if match and match.group(1) in _MODEL_IDS:+ return True++ # If the model is not a base OpenAI model or a fine-tuned OpenAI model, then+ # it is not an OpenAI model.+ return False+++#-------------------------------------------------------------------------------+# Function and classes required for the fine-tune interface+#-------------------------------------------------------------------------------++class FinetuneJobOpenAI(FinetuneJob):++ def __init__(self, *args, **kwargs):+ self.provider_file_id = None # TODO: Can we get this using the job_id?+ self.provider_job_id = None+ super().__init__(*args, **kwargs)++ def cancel(self):+ # Cancel the provider job+ if _does_job_exist(self.provider_job_id):+ status = _get_training_status(self.provider_job_id)+ if _is_terminal_training_status(status):+ err_msg = "Jobs that are complete cannot be canceled."+ err_msg += f" Job with ID {self.provider_job_id} is done."+ raise Exception(err_msg)+ openai.fine_tuning.jobs.cancel(self.provider_job_id)+ self.provider_job_id = None++ # Delete the provider file+ # TODO: Should there be a separate clean method?+ if self.provider_file_id is not None:+ if _does_file_exist(self.provider_file_id):+ openai.files.delete(self.provider_file_id)+ self.provider_file_id = None++ # Call the super's cancel method after the custom cancellation logic+ super().cancel()++ def status(self) -> TrainingStatus:+ status = _get_training_status(self.provider_job_id)+ return status+++def finetune_openai(+ job: FinetuneJobOpenAI,+ model: str,+ message_completion_pairs: List[Dict[str, str]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ ) -> Union[str, Exception]:+ train_kwargs = train_kwargs or {}+ train_data = message_completion_pairs+ train_method = TrainingMethod.SFT # Note: This could be an argument++ # Validate train data and method+ logger.info("[Finetune] Validating the formatting of the data")+ _validate_data(train_data, train_method)+ logger.info("[Finetune] Done!")++ # Convert to the OpenAI format+ logger.info("[Finetune] Converting the data to the OpenAI format")+ # TODO: Should we use the system prompt?+ train_data = _convert_data(train_data)+ logger.info("[Finetune] Done!")++ # Save to a file+ logger.info("[Finetune] Saving the data to a file")+ data_path = save_data(train_data, provider_name=PROVIDER_OPENAI)+ logger.info("[Finetune] Done!")++ # Upload the data to the cloud+ logger.info("[Finetune] Uploading the data to the provider")+ provider_file_id = _upload_data(data_path)+ job.provider_file_id = provider_file_id+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Start remote training")+ # We utilize model and train_kwargs here+ provider_job_id = _start_remote_training(+ train_file_id=job.provider_file_id,+ model=model,+ train_kwargs=train_kwargs,+ )+ job.provider_job_id = provider_job_id+ # job.provider_job_id = "ftjob-ZdEL1mUDk0dwdDuZJQOng8Vv"+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Wait for training to complete")+ # TODO: Would it be possible to stream the logs?+ _wait_for_job(job)+ logger.info("[Finetune] Done!")++ logger.info("[Finetune] Get trained model if the run was a success")+ model = _get_trained_model(job)+ logger.info("[Finetune] Done!")++ return model+++#-------------------------------------------------------------------------------+# Custom functions to support the finetune_* function+#-------------------------------------------------------------------------------++_SUPPORTED_TRAINING_METHODS = [+ TrainingMethod.SFT,+]++def _get_training_status(job_id: str) -> Union[TrainingStatus, Exception]:+ # TODO: Should this type be shared across all fine-tune clients?+ provider_status_to_training_status = {+ "validating_files": TrainingStatus.pending,+ "queued": TrainingStatus.pending,+ "running": TrainingStatus.running,+ "succeeded": TrainingStatus.succeeded,+ "failed": TrainingStatus.failed,+ "cancelled": TrainingStatus.cancelled,+ }++ # Check if there is an active job+ if job_id is None:+ logger.info("There is no active job.")+ return TrainingStatus.not_started++ err_msg = f"Job with ID {job_id} does not exist."+ assert _does_job_exist(job_id), err_msg++ # Retrieve the provider's job and report the status+ provider_job = openai.fine_tuning.jobs.retrieve(job_id)+ provider_status = provider_job.status+ status = provider_status_to_training_status[provider_status]++ return status+++def _does_job_exist(job_id: str) -> bool:+ try:+ # TODO: Error handling is vague+ openai.fine_tuning.jobs.retrieve(job_id)+ return True+ except Exception as e:+ return False+++def _does_file_exist(file_id: str) -> bool:+ try:+ # TODO: Error handling is vague+ openai.files.retrieve(file_id)+ return True+ except Exception as e:+ return False+++def _is_terminal_training_status(status: TrainingStatus) -> bool:+ return status in [+ TrainingStatus.succeeded,+ TrainingStatus.failed,+ TrainingStatus.cancelled,+ ]+++def _validate_data(+ data: Dict[str, str],+ train_method: TrainingMethod+ ) -> Optional[Exception]:+ # Check if this train method is supported+ if train_method not in _SUPPORTED_TRAINING_METHODS:+ err_msg = f"OpenAI does not support the training method {train_method}."+ raise ValueError(err_msg)++ validate_finetune_data(data, train_method)+++def _convert_data(+ data: List[Dict[str, str]],+ system_prompt: Optional[str]=None,+ ) -> Union[List[Dict[str, Any]], Exception]:+ # Item-wise conversion function+ def _row_converter(d):+ messages = [+ {"role": "user", "content": d["prompt"]},+ {"role": "assistant", "content": d["completion"]}+ ]+ if system_prompt:+ messages.insert(0, {"role": "system", "content": system_prompt})+ messages_dict = {"messages": messages}+ return messages_dict++ # Convert the data to the OpenAI format; validate the converted data+ converted_data = list(map(_row_converter, data))+ openai_data_validation(converted_data)+ return converted_data+++def _upload_data(data_path: str) -> str:+ # Upload the data to the provider+ provider_file = openai.files.create(+ file=open(data_path, "rb"),+ purpose="fine-tune",+ )+ return provider_file.id+++def _start_remote_training(+ train_file_id: str,+ model: id,+ train_kwargs: Optional[Dict[str, Any]]=None+ ) -> str:+ train_kwargs = train_kwargs or {}+ provider_job = openai.fine_tuning.jobs.create(+ model=model,+ training_file=train_file_id,+ hyperparameters=train_kwargs,+ )+ return provider_job.id+++def _wait_for_job(+ job: FinetuneJobOpenAI,+ poll_frequency: int=60,+):+ while not _is_terminal_training_status(job.status()):+ time.sleep(poll_frequency)+++def _get_trained_model(job):+ status = job.status()+ if status != TrainingStatus.succeeded:+ err_msg = f"Job status is {status}."+ err_msg += f" Must be {TrainingStatus.succeeded} to retrieve the model."+ logger.error(err_msg)+ raise Exception(err_msg)++ provider_job = openai.fine_tuning.jobs.retrieve(job.provider_job_id)+ finetuned_model = provider_job.fine_tuned_model+ return finetuned_model+++#-------------------------------------------------------------------------------+# OpenAI utility function(s)+#-------------------------------------------------------------------------------++# Adapted from https://cookbook.openai.com/examples/chat_finetuning_data_prep+def openai_data_validation(dataset: List[dict[str, Any]]):+ format_errors = defaultdict(int)+ for ex in dataset:+ if not isinstance(ex, dict):+ format_errors["data_type"] += 1+ continue++ messages = ex.get("messages", None)+ if not messages:+ format_errors["missing_messages_list"] += 1+ continue++ for message in messages:+ if "role" not in message or "content" not in message:+ format_errors["message_missing_key"] += 1++ if any(k not in ("role", "content", "name", "function_call", "weight") for k in message):+ format_errors["message_unrecognized_key"] += 1++ if message.get("role", None) not in ("system", "user", "assistant", "function"):+ format_errors["unrecognized_role"] += 1++ content = message.get("content", None)+ function_call = message.get("function_call", None)++ if (not content and not function_call) or not isinstance(content, str):+ format_errors["missing_content"] += 1++ if not any(message.get("role", None) == "assistant" for message in messages):+ format_errors["example_missing_assistant_message"] += 1++ # Raise an error if there are any format errors+ if format_errors:+ err_msg = "Found errors in the dataset format using the OpenAI API."+ err_msg += " Here are the number of datapoints for each error type:"+ for k, v in format_errors.items():+ err_msg += "\n {k}: {v}"+ raise ValueError(err_msg)diff --cc dspy/predict/predict.pyindex 88a74861a,a029a1171..259f3a3e9--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@@ -1,17 -1,15 +1,18 @@@+from __future__ import annotations # Used for self type hints++from functools import lru_cache+ import loggingimport random-from functools import lru_cache+from typing import Optionalfrom pydantic import BaseModelimport dspfrom dspy.predict.parameter import Parameter+from dspy.primitives.program import Modulefrom dspy.primitives.prediction import Prediction+ from dspy.primitives.program import Modulefrom dspy.signatures.signature import ensure_signature, signature_to_template- import logging- from functools import lru_cache@lru_cache(maxsize=None)def warn_once(msg: str):diff --cc dspy/primitives/module.pyindex 4fe9ea692,1dee9cbb1..87ce8b1aa--- a/dspy/primitives/module.py+++ b/dspy/primitives/module.py@@@ -149,11 -149,12 +149,11 @@@ class BaseModulereturn new_instancedef dump_state(self, save_verbose):- print(self.named_parameters())return {name: param.dump_state(save_verbose) for name, param in self.named_parameters()}- def load_state(self, state):+ def load_state(self, state, use_legacy_loading=False):for name, param in self.named_parameters():- param.load_state(state[name])+ param.load_state(state[name], use_legacy_loading=use_legacy_loading)def save(self, path, save_field_meta=False):with open(path, "w") as f:diff --cc dspy/teleprompt/finetune_teleprompter.pyindex 36544b34d,000000000..d5c5f563amode 100644,000000..100644--- a/dspy/teleprompt/finetune_teleprompter.py+++ b/dspy/teleprompt/finetune_teleprompter.py@@@ -1,415 -1,0 +1,415 @@@+from typing import Any, Callable, Dict, List, Optional, Union++import dspy- from dspy import logger++from dspy.utils.logging import logger+from dspy.evaluate.evaluate import Evaluate+from dspy.primitives.example import Example+from dspy.primitives.program import Program+from dspy.primitives.prediction import Prediction+from dspy.signatures.signature import signature_to_template+++#-------------------------------------------------------------------------------+# Templates for the user-facing strings used by this module+#-------------------------------------------------------------------------------++_INFO_DEFAULT_TEACHER = """No teacher provided. Using a copy of the student \+program as the teacher."""++_INFO_STRUCTURAL_EQUIVALENCY = """Ensuring that the student and teacher are \+are structurally equivalent."""++_INFO_SHARED_PREDICTOR = """Ensuring that the student and teacher programs do \+not share predictors."""++_INFO_LM_CONSISTENCY = """Ensuring that the teacher program satisfies the LM \+consistency property."""++_INFO_BOOTSTRAP_DATA = """Bootstrapping data on {} examples with the program \+{}, with {} threads"""++#-------------------------------------------------------------------------------+# Helper functions+#-------------------------------------------------------------------------------++# TODO: Shared below are useful functions. Similar procedures are implemented+# separately and used by other DSPy teleprompters. These can be moved to shared+# locations.+def prepare_teacher(+ student: Program,+ teacher: Program = None+ ) -> Union[Program, AssertionError]:+ """Prepare the teacher program with respect to the student program.++ Args:+ student: The student program.+ teacher: The teacher program. If `None`, a copy of the student program+ is used as the teacher. Defaults to `None`.++ Returns:+ The copied teacher program.++ Raises:+ AssertionError: If the teacher is not an instance of the Program class.+ """+ # If teacher is None, use a copy of the student program as the teacher+ if teacher is None:+ logger.info(_INFO_DEFAULT_TEACHER)+ teacher = student.deepcopy()+ else:+ teacher = teacher.deepcopy()++ # Ensure that the student and teacher programs have the same structure+ logger.info(_INFO_STRUCTURAL_EQUIVALENCY)+ student._assert_structural_equivalency(teacher)++ # Ensure that the predictors of the programs point to different objects+ logger.info(_INFO_SHARED_PREDICTOR)+ student._assert_no_shared_predictor(teacher)++ # Ensure that the LM consistency property is satisfied+ logger.info(_INFO_LM_CONSISTENCY)+ teacher._assert_lm_consistency()++ # If the global LM is being used, set it to the LMs of the copied teacher+ # program predictors to to avoid handling the same edge cases later+ if dspy.settings.lm:+ teacher._set_all_predictor_lms(dspy.settings.lm)++ return teacher+++def build_prompt_completion_data_from_trace(+ trace: List[Dict],+ exclude_demos: bool=False,+ try_to_record_lm_kwargs: bool = False,+ program: Program = None,+ ) -> Dict[str, List[Dict[str, Any]]]:+ """Build prompt completion data from a given trace.++ Args:+ trace: The trace from which the prompt-completion data will be built.+ exclude_demos: Exclude the demos from the prompts even if they are+ present in the trace. Defaults to `False`.+ try_to_record_lm_kwargs: Whether to record the LM kwargs in the data.+ Defaults to `False`. If set, the `lm_kwargs` field of the LM used to+ generate the prompt-completion pair is included in the data. To+ find the LM, we first check if the predictor that generated the+ prompt-completion pair has an LM field set (`predictor.lm`). If it+ does, we record it's kwargs. If it doesn't, we get the kwargs from+ `dspy.settings.lm`. If `dspy.settings.lm` is not set either, this+ function will not record the LM kwargs.+ program: Optional argument used to infer the name of the predictor that+ generated the prompt-completion pair. If provided, the returned data+ will include the `predictor_name` field. If not provided, the+ `predictor_name` field is not included in the data, but the caller+ of this function can recover this information by using the+ `predictor_ind` field that' included in the data by default, by+ building the following dictionary:++ {ind: n for ind, (n, _) in enumerate(program.named_predictors())}++ where `ind` is the index of the predictor in the list returned by+ the `named_predictors()` method of the program. Defaults to `None`.++ Returns:+ Data as a list of dictionaries with the keys `prompt`, `completion` and+ optionally with the keys `predictor_name` and `lm_kwargs`. For a given+ prompt-completion pair:+ - The `prompt` field corresponds to the prompt.+ - The `completion` field corresponds to the completion.+ - The `predictor_ind` field corresponds to the index of the predictor+ in the predictor list+ - The `predictor_name` field corresponds to the index of the predictor+ in the list returned by the the named_predictor() method of the+ program used to generate the trace. This field is included only if+ the `pred_ind_to_name` argument is provided.+ - The `lm_kwargs` field corresponds to the LM kwargs that generated the+ prompt-completion pair. Included only if the `record_lm_kwargs` is+ set and there is an active LM.+ """+ # If the program is provided, build the predictor index to name mapping+ if program:+ pred_ind_to_name = {+ ind: name for ind, (name, _) in enumerate(program.named_predictors())+ }++ # Build the prompt-completion data+ data = []+ for pred_ind, (pred, inputs, outputs) in enumerate(trace):+ # Get the demos from the predictor if exclude_demos is False+ demos = [] if exclude_demos else pred.demos++ # Build prompt and completion strings+ template = signature_to_template(pred.signature)+ prompt = template(Example(demos=demos, **inputs))+ completion = template.query(Example(**outputs))++ # TODO: This part of the code could be improved.+ # The method we use to build the completion (template.query) is meant to+ # be used for creating, well, queries, and hence contains field prefixes+ # (e.g. "Reasoning: Let's think step by step in order to"), which are+ # also contained in the last piece of the prompt (separated with a new+ # line) We remove this piece from the completion. This is a hacky+ # solution since it assumes a particular template format.+ prompt_last = prompt.split("\n")[-1]+ completion = completion[len(prompt_last):]++ # Create prompt-completion dictionary and add it to the data; optionally+ # add the predictor_name key as well as the lm_kwargs.+ data_dict = dict(prompt=prompt, completion=completion)++ # Record the predictor index and optionally, name+ data_dict['predictor_ind'] = pred_ind+ if program:+ data_dict['predictor_name'] = pred_ind_to_name[pred_ind]++ # Optionally, record the LM kwargs+ lm = pred.lm or dspy.settings.lm+ if try_to_record_lm_kwargs and lm:+ data_dict['lm_kwargs'] = lm.kwargs+ data.append(data_dict)++ return data+++def convert_to_module_level_prompt_completion_data(+ data: List[Dict],+ keep_data_keys: bool = False,+ exclude_demos: bool = False,+ try_to_record_lm_kwargs: bool = False,+ program: Program = None+ ) -> List[Dict]:+ """Convert the data to prompt-completion data using the "trace" field.++ This function is a wrapper around the function+ `build_prompt_completion_data_from_trace`, calling it on the "trace" field+ of each dictionary in the input data list and combiningin the results into+ a list of prompt-completion data dictionaries. If the `keep_data_keys`+ is set, the original data keys are also copied over to in the+ prompt-completion dictionaries.++ For example, if the input data includes 10 dictionaries, each containing a+ "trace" field generated by a program with 3 predictors, the returned data+ will have 30 prompt-completion data dictionaries.++ Args:+ data: List of data dictionaries to be converted to prompt-completion+ data. Each dictionary in the list should contain a "trace" field,+ which is passed to the `build_prompt_completion_data_from_trace`+ function to generate the prompt-completion data.+ keep_data_keys: Whether to keep the original data keys in the+ prompt-completion data. Note that if there are keys that are common+ between the original data and the prompt-completion data returned by+ `build_prompt_completion_data_from_trace`, the values from the+ prompt-completion data will overwrite the values from the original+ data. Defaults to `False`.+ exclude_demos: Passed to `build_prompt_completion_data_from_trace`.+ Defaults to `False`.+ try_to_record_lm_kwargs: Passed to+ `build_prompt_completion_data_from_trace`. Defaults to `False`.+ program: Passed to `build_prompt_completion_data_from_trace`.+ Defaults to `None`.+ """+ prompt_completion_data = []+ for data_dict in data:+ trace = data_dict["trace"]+ trace_prompt_comletion_data = build_prompt_completion_data_from_trace(+ trace=trace, exclude_demos=exclude_demos,+ try_to_record_lm_kwargs=try_to_record_lm_kwargs, program=program+ )+ for prompt_completion_dict in trace_prompt_comletion_data:+ if keep_data_keys:+ prompt_completion_dict = {**data_dict, **prompt_completion_dict}+ prompt_completion_data.append(prompt_completion_dict)+ return prompt_completion_data+++def bootstrap_data(+ program: Program,+ dataset: List[Example],+ metric: Optional[Callable[+ [Example, Prediction, Optional[List]], Union[bool, int, float]+ ]] = None,+ num_threads = 1,+ ) -> List[Dict[str, Any]]:+ """Bootstrap prediction and trace data for the program using the dataset.++ Args:+ program: The program that will be used to generate the traces for data+ collection.+ dataset: The dataset to be used for data collection.+ metric: The optional metric to be used to get a score for the example,+ recorded in a `score` field in the data. If the metric is not+ provided, the `score` field is not included in the data. Defaults+ to `None`.+ num_threads: The number of threads to be used for data collection.+ Defaults to `1`.++ Returns:+ Data as a list of dictionaries with the keys `example`, `prediction`,+ `trace`, and optionally, `score` fields. For a given example:+ - The `example` field corresponds to the example itself.+ - The `example_ind` field corresponds to the index of the example in the+ `dataset`.+ - The `prediction` field corresponds to the prediction made by the+ program on the example.+ - The `trace` field corresponds to the trace generated by the program+ on the example.+ - The `score` field corresponds to the metric score of the example, if+ the metric is provided. Otherwise, it is not included in the data.+ """+ data = []++ # Use Evaluate to call the program have the responses cached+ cname = program.__class__.__name__+ info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)+ logger.info(info)+ evaluator = Evaluate(+ devset=dataset, num_threads=num_threads, display_progress=True+ )+ evaluator(program, metric=metric)++ # Re-iterate over the dataset to build the cached prompt-completion data+ for example_ind, example in enumerate(dataset):++ # Run the program on the example+ with dspy.context(trace=[]):+ prediction = program(**example.inputs())+ trace = dspy.settings.trace+ score = metric(example, prediction, trace) if metric else None++ # Build the data dictionary and extend the data list+ data_dict = dict(example=example, prediction=prediction, trace=trace)+ data_dict['example_ind'] = example_ind+ if metric:+ data_dict['score'] = score+ data.append(data_dict)++ return data+++# TODO: If we can ensure to pass the "round" information every time a call is+# issued to an LM, we can make repetitive un-cached calls to the same LM without+# modifying it's temperature. This function can be removed then.+def bootstrap_data_for_round(+ program: Program,+ dataset: List[Example],+ metric: Optional[Callable[+ [Example, Prediction, Optional[List]], Union[bool, int, float]+ ]] = None,+ num_threads = 1,+ sampling_round: int = 0,+ sampling_temperature: Optional[float] = 0.9,+ sampling_temperature_delta: float = 0.001,+ ) -> Union[List[Dict], AssertionError]:+ """ Bootstrap data for the given sampling round.++ This is a wrapper function around the `bootstrap_data` function that allows+ for collecting data for the given `sampling_round`. Due to the way caching+ works, one cannot get different completions for the same prompt just by+ querying an LM again, despite using a high sampling temperature. This+ function is a workaround to get different completions for the same prompt+ by modifying the sampling temperature of the LM for the specified+ `sampling_round`. The temperature of an LM is set to the following value+ for the given `sampling_round`:++ sampling_temperature + sampling_temperature_delta * sampling_round++ If a `sampling_temperature` of `None` is passed, the already set temperature+ of the LM is used as the sampling temperature instead.++ To sample different completions for the same prompt, this sampling can be+ called multiple times with different `sampling_round` values. For example:+ ```+ num_rounds = 5+ data = []+ for round in range(num_rounds):+ data += bootstrap_data_for_round(+ program, dataset, metric=metric, sampling_round=round+ )+ # Any dataset filtering for the next round can be done here, if needed+ ```++ The dataset filtering can become a powerful tool. For example, it can be+ used to filter out the examples that have already had enough high scoring+ completions generated for them. Here is an illustration:+ ```+ num_correct_target = 3+ correct_score = 1+ num_rounds = 5+ data = []+ for round in range(num_rounds):+ data += bootstrap_data_for_round(+ program, dataset, metric=metric, sampling_round=round+ )+ correct_bootstraps = [d for d in data if d['score'] == correct_score]+ correct_counts = Counter([d['example'] for d in correct_bootstraps])+ dataset = [d for d in dataset if correct_counts[d] < num_correct_target]+ ```++ Args:+ program: The program that will be used to generate the traces for data+ collection.+ dataset: The dataset to be used for data collection.+ metric: The optional metric to be used to get a score for the example,+ recorded in a `score` field in the data. If the metric is not+ provided, the `score` field is not included in the data. Defaults to+ `None`.+ num_threads: The number of threads to be used for data collection.+ Defaults to `1`.+ sampling_round: The round index for which the data is being collected.+ Defaults to `0`.+ sampling_temperature: The sampling temperature to be used for round+ `0`. If a value of `None` is passed, the temperature of the LM is+ used as the sampling temperature instead. Defaults to a high+ temperature of `0.9`.+ sampling_temperature_delta: The small temperature delta that's added to+ the sampling temperature every increment of the round index to+ generate different completions for the same prompt. Defaults to+ `0.001`.++ Returns:+ Data as a list of dictionaries with the keys returned by+ the `bootstrap_data` function, descriptions for which are shared in the+ function documentation. This function adds the following extra field+ to the dictionaries:+ - The `round` field corresponds to the `sampling_round` argument.+ """+ # TODO: [DSPy 2.5] Migrate to using "seed" for sampling+ # Helper function to adjust the temperature of the LM. If a None temperature+ # is passed, keep the LM's temperature as is as the base temperature, then+ # adjust the temperature for the given round.+ def copy_model_with_updated_temp(lm):+ temp = sampling_temperature+ temp = lm.kwargs["temperature"] if temp is None else temp+ temp = temp + sampling_temperature_delta * sampling_round+ return lm.copy(temperature=temp)++ # Ensure that the LM consistency is satisfied, which ensures that either (1)+ # the global LM is set or (2) all the predictors have an LM set.+ program._assert_lm_consistency()++ # Deepcopy the program and copy the dataset to avoid modifying the original+ program = program.deepcopy()+ dataset = dataset.copy()++ # Update the temperature of the LM for the given round+ context_lm = None+ if dspy.settings.lm:+ context_lm = copy_model_with_updated_temp(dspy.settings.lm)+ else:+ for pred in program.predictors():+ pred.lm = copy_model_with_updated_temp(pred.lm)++ # Collect the data for the given round+ with dspy.context(lm=context_lm):+ data = bootstrap_data(+ program, dataset, metric=metric, num_threads=num_threads+ )++ # Add the round information to the data+ for data_dict in data:+ data_dict["round"] = sampling_round++ return data