Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Merge main

dilarae7d5046eac1e
diff --cc dspy/clients/anyscale.py
index 43220f6b6,000000000..dea3ca4c5
mode 100644,000000..100644
--- a/dspy/clients/anyscale.py
+++ b/dspy/clients/anyscale.py
@@@ -1,109 -1,0 +1,107 @@@
- import re
+import time
+from typing import Any, Dict, List, Optional
+
- import dspy
- from dspy import logger
++from dspy.utils.logging import logger
+from dspy.clients.finetune import (
+ FinetuneJob,
+ TrainingMethod,
+ get_finetune_directory,
+ validate_finetune_data,
+)
+
+#-------------------------------------------------------------------------------
+# Variables
+#-------------------------------------------------------------------------------
+
+# List of training methods supported by AnyScale
+TRAINING_METHODS_ANYSCALE = [
+ TrainingMethod.SFT,
+]
+
+#-------------------------------------------------------------------------------
+# Launching and killing LMs
+#-------------------------------------------------------------------------------
+
+def anyscale_model_launch(model: str, launch_kwargs: Dict[str, Any]):
+ """Launch an AnyScale model."""
+ # TODO: Hardcode resources for launching a select server through docker
+ raise NotImplementedError("Method `anyscale_model_launch` is not implemented.")
+
+
+def anyscale_model_kill(model: str, launch_kwargs: Dict[str, Any]):
+ # TODO: Hardcode resources for killing a select server through docker
+ """Kill an AnyScale model."""
+ raise NotImplementedError("Method `anyscale_model_kill` is not implemented.")
+
+
+#-------------------------------------------------------------------------------
+# Function and classes required for the fine-tune interface
+#-------------------------------------------------------------------------------
+
+class FinetuneJobAnyScale(FinetuneJob):
+
+ def cancel(self):
+ """Cancel the finetune job."""
+ raise NotImplementedError("Method `cancel` is not implemented.")
+ # Call the super's cancel method after the custom cancellation logic,
+ # so that the future can be cancelled
+ # super().cancel()
+
+ def status(self):
+ """Get the status of the finetune job."""
+ raise NotImplementedError("Method `status` is not implemented.")
+
+
+def is_anyscale_model(model: str) -> bool:
+ """Check if the model is an AnyScale model."""
+ logger.info("Is AnyScale model is not implemented, returning False as a default to not break lm.py")
+ return False
+
+
+def finetune_anyscale(
+ job: FinetuneJobAnyScale,
+ model: str,
+ message_completion_pairs: List[Dict[str, str]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+ ) -> str:
+ """Fine-tune with AnyScale."""
+ # Fake fine-tuning
+ logger.info("[Finetune] Fake fine-tuning")
+ train_kwargs = train_kwargs or {}
+
+ try:
+ logger.info("[Finetune] Validate the formatting of the fine-tuning data")
+ training_method = TrainingMethod.SFT # Hardcoded
+ validate_finetune_data(message_completion_pairs, training_method)
+ time.sleep(5)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Saving the data to a file")
+ finetune_parent_dir = get_finetune_directory()
+ # We utilize message_completion_pairs here
+ time.sleep(1)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Uploading the data to the cloud")
+ time.sleep(1)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Launch training")
+ # We utilize model and train_kwargs here
+ time.sleep(1)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Wait for training to complete")
+ time.sleep(1)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Get trained model client")
+ model = "anyscale_model" # Hardcoded
+ time.sleep(1)
+ logger.info("[Finetune] Done!")
+
+ return model
+
+ except Exception as e:
+ logger.error(f"[Finetune] Error: {e}")
+ raise e
diff --cc dspy/clients/finetune.py
index a6e6e0005,000000000..3c14be363
mode 100644,000000..100644
--- a/dspy/clients/finetune.py
+++ b/dspy/clients/finetune.py
@@@ -1,125 -1,0 +1,123 @@@
+from abc import abstractmethod
+from concurrent.futures import Future
+from enum import Enum
+import os
+from typing import List, Dict, Any, Optional
+import ujson
+
+from datasets.fingerprint import Hasher
+
- from dspy import logger
-
+
+# TODO: Move to a centralized location with all the other environment variables
+# Set the directory to save the fine-tuned models
+def get_finetune_directory() -> str:
+ """Get the directory to save the fine-tuned models."""
+ alternative_path = os.path.join(os.getcwd(), '.dspy_finetune')
+ return os.environ.get('DSPY_FINETUNEDIR') or alternative_path
+
+
+FINETUNE_DIRECTORY = get_finetune_directory()
+
+
+class TrainingMethod(str, Enum):
+ """Enum class for training methods.
+
+ When comparing enums, Python checks for object IDs, which means that the
+ enums can't be compared directly. Subclassing the Enum class along with the
+ str class allows for direct comparison of the enums.
+ """
+ SFT = "SFT"
+ Preference = "Preference"
+
+
+class TrainingStatus(str, Enum):
+ """Enum class for remote training status."""
+ not_started = "not_started"
+ pending = "pending"
+ running = "running"
+ succeeded = "succeeded"
+ failed = "failed"
+ cancelled = "cancelled"
+
+
+"""Dictionary mapping training methods to the data keys they require."""
+TRAINING_METHOD_TO_DATA_KEYS = {
+ TrainingMethod.SFT: ["prompt", "completion"],
+ TrainingMethod.Preference: ["prompt", "chosen", "rejected"],
+}
+
+
+class FinetuneJob(Future):
+
+ def __init__(self,
+ model: str,
+ message_completion_pairs: List[Dict[str, str]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+ ):
+ self.model = model
+ self.message_completion_pairs = message_completion_pairs
+ self.train_kwargs: Dict[str, Any] = train_kwargs or {}
+ super().__init__()
+
+ def get_kwargs(self):
+ return dict(
+ model=self.model,
+ message_completion_pairs=self.message_completion_pairs,
+ train_kwargs=self.train_kwargs,
+ )
+
+ def __str__(self):
+ return f"FinetuningJob({self.to_dict()})"
+
+ def __repr__(self):
+ return str(self)
+
+ # Subclasses should override the cancel method to cancel the finetune job;
+ # then call the super's cancel method so that the future can be cancelled.
+ def cancel(self):
+ """Cancel the finetune job."""
+ super().cancel()
+
+ @abstractmethod
+ def status(self):
+ """Get the status of the finetune job."""
+ raise NotImplementedError("Method `status` is not implemented.")
+
+
+def validate_finetune_data(
+ data: List[Dict[str, Any]],
+ train_method: TrainingMethod
+ ) -> Optional[AssertionError]:
+ """Validate the finetune data based on the training method."""
+ # Get the required data keys for the training method
+ required_keys = TRAINING_METHOD_TO_DATA_KEYS[train_method]
+
+ # Check if the training data has the required keys
+ for ind, data_dict in enumerate(data):
+ err_msg = f"The datapoint at index {ind} is missing the keys required for {train_method} training."
+ err_msg = f"\n Expected: {required_keys}"
+ err_msg = f"\n Found: {data_dict.keys()}"
+ assert all([key in data_dict for key in required_keys]), err_msg
+
+
+def save_data(
+ data: List[Dict[str, Any]],
+ provider_name: Optional[str]=None,
+ ) -> str:
+ """Save the fine-tuning data to a file."""
+ # Construct the file name based on the data hash
+ hash = Hasher.hash(data)
+ file_name = f"{hash}.jsonl"
+ file_name = f"{provider_name}_{file_name}" if provider_name else file_name
+
+ # Find the directory to save the fine-tuning data
+ finetune_parent_dir = get_finetune_directory()
+ os.makedirs(finetune_parent_dir, exist_ok=True)
+
+ # Save the data to a file
+ file_path = os.path.join(finetune_parent_dir, file_name)
+ file_path = os.path.abspath(file_path)
+ with open(file_path, "w") as f:
+ for item in data:
+ f.write(ujson.dumps(item) + "\n")
+ return file_path
diff --cc dspy/clients/lm.py
index b72fc41f7,994c8bc41..014025efc
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@@ -1,62 -1,23 +1,53 @@@
+from concurrent.futures import ThreadPoolExecutor
++from datetime import datetime
import functools
import os
-import uuid
-from datetime import datetime
from pathlib import Path
+from typing import Any, Dict, List, Optional, Type, Union
+import ujson
++import uuid
+
+
- from dspy import logger
++from dspy.utils.logging import logger
+from dspy.clients.finetune import FinetuneJob
+from dspy.clients.self_hosted import (
+ is_self_hosted_model,
+ self_hosted_model_launch,
+ self_hosted_model_kill,
+)
+from dspy.clients.anyscale import (
+ is_anyscale_model,
+ anyscale_model_launch,
+ anyscale_model_kill,
+)
+ import litellm
-import ujson
+ from litellm.caching import Cache
- DISK_CACHE_DIR = os.environ.get('DSPY_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache')
-disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
-litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")
++DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
++litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
+ litellm.telemetry = False
-
- try:
- import warnings
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
- os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
- import litellm
- litellm.telemetry = False
-
- from litellm.caching import Cache
- litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
- except ImportError:
- class LitellmPlaceholder:
- def __getattr__(self, _): raise ImportError("The LiteLLM package is not installed. Run `pip install litellm`.")
-
- litellm = LitellmPlaceholder()
+
+#-------------------------------------------------------------------------------
+# LiteLLM Client
+#-------------------------------------------------------------------------------
+
class LM:
- def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs):
+ def __init__(self,
+ model,
+ model_type='chat',
+ temperature=0.0,
+ max_tokens=1000,
+ cache=True,
+ launch_kwargs=None,
+ **kwargs
+ ):
+ # Remember to update LM.copy() if you modify the constructor!
self.model = model
self.model_type = model_type
self.cache = cache
@@@ -64,10 -24,10 +55,11 @@@
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
self.history = []
+ # TODO: This is error prone!
if "o1-" in model:
- assert max_tokens >= 5000 and temperature == 1.0, \
- "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"
+ assert (
+ max_tokens >= 5000 and temperature == 1.0
+ ), "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"
def __call__(self, prompt=None, messages=None, **kwargs):
# Build the request.
@@@ -87,22 -49,17 +81,29 @@@
entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response)
entry = dict(**entry, outputs=outputs, usage=dict(response["usage"]))
entry = dict(**entry, cost=response.get("_hidden_params", {}).get("response_cost"))
+ entry = dict(
+ **entry,
+ timestamp=datetime.now().isoformat(),
+ uuid=str(uuid.uuid4()),
+ model=self.model,
+ model_type=self.model_type,
+ )
self.history.append(entry)
-
+
return outputs
+ def copy(self, **kwargs):
+ """Create a copy of the current LM with updated kwargs."""
+ kwargs = {
+ "model": self.model,
+ "model_type": self.model_type,
+ "cache": self.cache,
+ "launch_kwargs": self.launch_kwargs,
+ **self.kwargs,
+ **kwargs,
+ }
+ return self.__class__(**kwargs)
+
def inspect_history(self, n: int = 1):
_inspect_history(self, n)
@@@ -221,136 -133,5 +236,136 @@@ def _inspect_history(lm, n: int = 1)
if len(outputs) > 1:
choices_text = f" \t (and {len(outputs)-1} other completions)"
print(_red(choices_text, end=""))
-
+
print("\n\n\n")
+
+
+#-------------------------------------------------------------------------------
+# Fine-tune support
+#-------------------------------------------------------------------------------
+# TODO: This part can be moved to a separate module
+
+from dspy.clients.openai import (
+ FinetuneJobOpenAI,
+ is_openai_model,
+ finetune_openai
+)
+from dspy.clients.anyscale import (
+ FinetuneJobAnyScale,
+ is_anyscale_model,
+ finetune_anyscale
+)
+
+
+_PROVIDER_ANYSCALE = "anyscale"
+_PROVIDER_OPENAI = "openai"
+_SUPPORTED_FINETUNE_PROVIDERS = [
+ _PROVIDER_ANYSCALE,
+ _PROVIDER_OPENAI,
+]
+
+
+def _get_supported_finetune_provider(model: str) -> Union[str, ValueError]:
+ """Return the finetuning provider for the given model.
+
+ The provider must be in _SUPPORTED_FINETUNE_PROVIDERS. This function is not
+ named `_get_provider` because it does not attempt to find the provider of a
+ model if there is no DSPy fine-tuning support for it.
+ """
+ if is_openai_model(model):
+ return _PROVIDER_OPENAI
+
+ if is_anyscale_model(model):
+ return _PROVIDER_ANYSCALE
+
+ return ValueError(f"DSPy does not have fine-tuning support for {model}")
+
+
+def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:
+ """Get the FinetuneJob class for the provider."""
+ # Mapping from provider to finetune job type
+ _PROVIDER_TO_FINETUNE_JOB_CLASS = {
+ _PROVIDER_ANYSCALE: FinetuneJobAnyScale,
+ _PROVIDER_OPENAI: FinetuneJobOpenAI,
+ }
+
+ # Get the FinetuneJob class for the provider
+ _CLS = _PROVIDER_TO_FINETUNE_JOB_CLASS[provider]
+
+ return _CLS
+
+
+def get_provider_finetune_function(provider: str) -> callable:
+ """Return the finetune function for the given model."""
+ # Mapping from provider to finetune function
+ _PROVIDER_TO_FINETUNE_FUNCTION = {
+ _PROVIDER_ANYSCALE: finetune_anyscale,
+ _PROVIDER_OPENAI: finetune_openai,
+ }
+
+ # Get the finetuning provider
+ finetune_function = _PROVIDER_TO_FINETUNE_FUNCTION[provider]
+
+ return finetune_function
+
+
+def execute_finetune_job(
+ job: FinetuneJob[Type[LM]],
+ lm: LM,
+ cache_finetune: bool=True
+):
+ """Execute the finetune job in a blocking manner."""
+ try:
+ job_kwargs = job.get_kwargs()
+ if cache_finetune:
+ model = cached_finetune(job=job, **job_kwargs)
+ else:
+ model = finetune(job=job, **job_kwargs)
+ lm = lm.copy(model=model)
+ job.set_result(lm)
+ except Exception as err:
+ logger.error(err)
+ job.set_result(err)
+
+
+# TODO: Perhaps we shouldn't directly cache the data
+# TODO: Add DiskCache, ignore job
+def cached_finetune(
+ job,
+ model: str,
+ message_completion_pairs: List[Dict[str, str]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+) -> Union[str, ValueError]:
+ return finetune(
+ job=job,
+ model=model,
+ message_completion_pairs=message_completion_pairs,
+ train_kwargs=train_kwargs,
+ )
+
+
+def finetune(
+ job,
+ model: str,
+ message_completion_pairs: List[Dict[str, str]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+) -> Union[str, Exception]:
+ """Fine-tune a new model based on the given model."""
+ # Get the fine-tuning provider
+ try:
+ provider = _get_supported_finetune_provider(model)
+
+ # Get the finetune function
+ provider_finetune_function = get_provider_finetune_function(provider)
+
+ # Fine-tune a new model based on the given model
+ model = provider_finetune_function(
+ job=job,
+ model=model,
+ message_completion_pairs=message_completion_pairs,
+ train_kwargs=train_kwargs,
+ )
+ except Exception as err:
+ raise err
+
+ return model
diff --cc dspy/clients/openai.py
index 95af1f16f,000000000..f7feb961a
mode 100644,000000..100644
--- a/dspy/clients/openai.py
+++ b/dspy/clients/openai.py
@@@ -1,367 -1,0 +1,367 @@@
+from collections import defaultdict
+import re
+import time
+from typing import Any, Dict, List, Optional, Union
+
+import openai
+
- from dspy import logger
++from dspy.utils.logging import logger
+from dspy.clients.finetune import (
+ FinetuneJob,
+ TrainingMethod,
+ TrainingStatus,
+ validate_finetune_data,
+ save_data,
+)
+
+
+#-------------------------------------------------------------------------------
+# Function and variables to check if a model is an OpenAI model
+#-------------------------------------------------------------------------------
+
+# Provider name
+PROVIDER_OPENAI = "openai"
+
+# List of model IDs
+_MODEL_IDS = [
+ "gpt-4o",
+ "gpt-4o-2024-08-06",
+ "gpt-4o-2024-05-13",
+ "chatgpt-4o-latest",
+ "gpt-4o-mini",
+ "gpt-4o-mini-2024-07-18",
+ "gpt-4o-realtime-preview",
+ "gpt-4o-realtime-preview-2024-10-01",
+ "o1-preview",
+ "o1-preview-2024-09-12",
+ "o1-mini",
+ "o1-mini-2024-09-12",
+ "gpt-4-turbo",
+ "gpt-4-turbo-2024-04-09",
+ "gpt-4-turbo-preview",
+ "gpt-4-0125-preview",
+ "gpt-4-1106-preview",
+ "gpt-4",
+ "gpt-4-0613",
+ "gpt-4-0314",
+ "gpt-3.5-turbo-0125",
+ "gpt-3.5-turbo",
+ "gpt-3.5-turbo-1106",
+ "gpt-3.5-turbo-instruct",
+ "dall-e-3",
+ "dall-e-2",
+ "tts-1",
+ "tts-1-hd",
+ "text-embedding-3-large",
+ "text-embedding-3-small",
+ "text-embedding-ada-002",
+ "omni-moderation-latest",
+ "omni-moderation-2024-09-26",
+ "text-moderation-latest",
+ "text-moderation-stable",
+ "text-moderation-007",
+ "babbage-002",
+ "davinci-002"
+]
+
+
+def is_openai_model(model: str) -> bool:
+ """Check if the model is an OpenAI model."""
+ # Filter the provider_prefix, if exists
+ provider_prefix = f"{PROVIDER_OPENAI}/"
+ if model.startswith(provider_prefix):
+ model = model[len(provider_prefix):]
+
+ # Check if the model is a base OpenAI model
+ if model in _MODEL_IDS:
+ return True
+
+ # Check if the model is a fine-tuned OpneAI model. Fine-tuned OpenAI models
+ # have the prefix "ft:<BASE_MODEL_NAME>:", followed by a string specifying
+ # the fine-tuned model. The following RegEx pattern is used to match the
+ # base model name.
+ # TODO: This part can be updated to match the actual fine-tuned model names
+ # by making a call to the OpenAI API to be more exact, but this might
+ # require an API key with the right permissions.
+ match = re.match(r"ft:([^:]+):", model)
+ if match and match.group(1) in _MODEL_IDS:
+ return True
+
+ # If the model is not a base OpenAI model or a fine-tuned OpenAI model, then
+ # it is not an OpenAI model.
+ return False
+
+
+#-------------------------------------------------------------------------------
+# Function and classes required for the fine-tune interface
+#-------------------------------------------------------------------------------
+
+class FinetuneJobOpenAI(FinetuneJob):
+
+ def __init__(self, *args, **kwargs):
+ self.provider_file_id = None # TODO: Can we get this using the job_id?
+ self.provider_job_id = None
+ super().__init__(*args, **kwargs)
+
+ def cancel(self):
+ # Cancel the provider job
+ if _does_job_exist(self.provider_job_id):
+ status = _get_training_status(self.provider_job_id)
+ if _is_terminal_training_status(status):
+ err_msg = "Jobs that are complete cannot be canceled."
+ err_msg += f" Job with ID {self.provider_job_id} is done."
+ raise Exception(err_msg)
+ openai.fine_tuning.jobs.cancel(self.provider_job_id)
+ self.provider_job_id = None
+
+ # Delete the provider file
+ # TODO: Should there be a separate clean method?
+ if self.provider_file_id is not None:
+ if _does_file_exist(self.provider_file_id):
+ openai.files.delete(self.provider_file_id)
+ self.provider_file_id = None
+
+ # Call the super's cancel method after the custom cancellation logic
+ super().cancel()
+
+ def status(self) -> TrainingStatus:
+ status = _get_training_status(self.provider_job_id)
+ return status
+
+
+def finetune_openai(
+ job: FinetuneJobOpenAI,
+ model: str,
+ message_completion_pairs: List[Dict[str, str]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+ ) -> Union[str, Exception]:
+ train_kwargs = train_kwargs or {}
+ train_data = message_completion_pairs
+ train_method = TrainingMethod.SFT # Note: This could be an argument
+
+ # Validate train data and method
+ logger.info("[Finetune] Validating the formatting of the data")
+ _validate_data(train_data, train_method)
+ logger.info("[Finetune] Done!")
+
+ # Convert to the OpenAI format
+ logger.info("[Finetune] Converting the data to the OpenAI format")
+ # TODO: Should we use the system prompt?
+ train_data = _convert_data(train_data)
+ logger.info("[Finetune] Done!")
+
+ # Save to a file
+ logger.info("[Finetune] Saving the data to a file")
+ data_path = save_data(train_data, provider_name=PROVIDER_OPENAI)
+ logger.info("[Finetune] Done!")
+
+ # Upload the data to the cloud
+ logger.info("[Finetune] Uploading the data to the provider")
+ provider_file_id = _upload_data(data_path)
+ job.provider_file_id = provider_file_id
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Start remote training")
+ # We utilize model and train_kwargs here
+ provider_job_id = _start_remote_training(
+ train_file_id=job.provider_file_id,
+ model=model,
+ train_kwargs=train_kwargs,
+ )
+ job.provider_job_id = provider_job_id
+ # job.provider_job_id = "ftjob-ZdEL1mUDk0dwdDuZJQOng8Vv"
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Wait for training to complete")
+ # TODO: Would it be possible to stream the logs?
+ _wait_for_job(job)
+ logger.info("[Finetune] Done!")
+
+ logger.info("[Finetune] Get trained model if the run was a success")
+ model = _get_trained_model(job)
+ logger.info("[Finetune] Done!")
+
+ return model
+
+
+#-------------------------------------------------------------------------------
+# Custom functions to support the finetune_* function
+#-------------------------------------------------------------------------------
+
+_SUPPORTED_TRAINING_METHODS = [
+ TrainingMethod.SFT,
+]
+
+def _get_training_status(job_id: str) -> Union[TrainingStatus, Exception]:
+ # TODO: Should this type be shared across all fine-tune clients?
+ provider_status_to_training_status = {
+ "validating_files": TrainingStatus.pending,
+ "queued": TrainingStatus.pending,
+ "running": TrainingStatus.running,
+ "succeeded": TrainingStatus.succeeded,
+ "failed": TrainingStatus.failed,
+ "cancelled": TrainingStatus.cancelled,
+ }
+
+ # Check if there is an active job
+ if job_id is None:
+ logger.info("There is no active job.")
+ return TrainingStatus.not_started
+
+ err_msg = f"Job with ID {job_id} does not exist."
+ assert _does_job_exist(job_id), err_msg
+
+ # Retrieve the provider's job and report the status
+ provider_job = openai.fine_tuning.jobs.retrieve(job_id)
+ provider_status = provider_job.status
+ status = provider_status_to_training_status[provider_status]
+
+ return status
+
+
+def _does_job_exist(job_id: str) -> bool:
+ try:
+ # TODO: Error handling is vague
+ openai.fine_tuning.jobs.retrieve(job_id)
+ return True
+ except Exception as e:
+ return False
+
+
+def _does_file_exist(file_id: str) -> bool:
+ try:
+ # TODO: Error handling is vague
+ openai.files.retrieve(file_id)
+ return True
+ except Exception as e:
+ return False
+
+
+def _is_terminal_training_status(status: TrainingStatus) -> bool:
+ return status in [
+ TrainingStatus.succeeded,
+ TrainingStatus.failed,
+ TrainingStatus.cancelled,
+ ]
+
+
+def _validate_data(
+ data: Dict[str, str],
+ train_method: TrainingMethod
+ ) -> Optional[Exception]:
+ # Check if this train method is supported
+ if train_method not in _SUPPORTED_TRAINING_METHODS:
+ err_msg = f"OpenAI does not support the training method {train_method}."
+ raise ValueError(err_msg)
+
+ validate_finetune_data(data, train_method)
+
+
+def _convert_data(
+ data: List[Dict[str, str]],
+ system_prompt: Optional[str]=None,
+ ) -> Union[List[Dict[str, Any]], Exception]:
+ # Item-wise conversion function
+ def _row_converter(d):
+ messages = [
+ {"role": "user", "content": d["prompt"]},
+ {"role": "assistant", "content": d["completion"]}
+ ]
+ if system_prompt:
+ messages.insert(0, {"role": "system", "content": system_prompt})
+ messages_dict = {"messages": messages}
+ return messages_dict
+
+ # Convert the data to the OpenAI format; validate the converted data
+ converted_data = list(map(_row_converter, data))
+ openai_data_validation(converted_data)
+ return converted_data
+
+
+def _upload_data(data_path: str) -> str:
+ # Upload the data to the provider
+ provider_file = openai.files.create(
+ file=open(data_path, "rb"),
+ purpose="fine-tune",
+ )
+ return provider_file.id
+
+
+def _start_remote_training(
+ train_file_id: str,
+ model: id,
+ train_kwargs: Optional[Dict[str, Any]]=None
+ ) -> str:
+ train_kwargs = train_kwargs or {}
+ provider_job = openai.fine_tuning.jobs.create(
+ model=model,
+ training_file=train_file_id,
+ hyperparameters=train_kwargs,
+ )
+ return provider_job.id
+
+
+def _wait_for_job(
+ job: FinetuneJobOpenAI,
+ poll_frequency: int=60,
+):
+ while not _is_terminal_training_status(job.status()):
+ time.sleep(poll_frequency)
+
+
+def _get_trained_model(job):
+ status = job.status()
+ if status != TrainingStatus.succeeded:
+ err_msg = f"Job status is {status}."
+ err_msg += f" Must be {TrainingStatus.succeeded} to retrieve the model."
+ logger.error(err_msg)
+ raise Exception(err_msg)
+
+ provider_job = openai.fine_tuning.jobs.retrieve(job.provider_job_id)
+ finetuned_model = provider_job.fine_tuned_model
+ return finetuned_model
+
+
+#-------------------------------------------------------------------------------
+# OpenAI utility function(s)
+#-------------------------------------------------------------------------------
+
+# Adapted from https://cookbook.openai.com/examples/chat_finetuning_data_prep
+def openai_data_validation(dataset: List[dict[str, Any]]):
+ format_errors = defaultdict(int)
+ for ex in dataset:
+ if not isinstance(ex, dict):
+ format_errors["data_type"] += 1
+ continue
+
+ messages = ex.get("messages", None)
+ if not messages:
+ format_errors["missing_messages_list"] += 1
+ continue
+
+ for message in messages:
+ if "role" not in message or "content" not in message:
+ format_errors["message_missing_key"] += 1
+
+ if any(k not in ("role", "content", "name", "function_call", "weight") for k in message):
+ format_errors["message_unrecognized_key"] += 1
+
+ if message.get("role", None) not in ("system", "user", "assistant", "function"):
+ format_errors["unrecognized_role"] += 1
+
+ content = message.get("content", None)
+ function_call = message.get("function_call", None)
+
+ if (not content and not function_call) or not isinstance(content, str):
+ format_errors["missing_content"] += 1
+
+ if not any(message.get("role", None) == "assistant" for message in messages):
+ format_errors["example_missing_assistant_message"] += 1
+
+ # Raise an error if there are any format errors
+ if format_errors:
+ err_msg = "Found errors in the dataset format using the OpenAI API."
+ err_msg += " Here are the number of datapoints for each error type:"
+ for k, v in format_errors.items():
+ err_msg += "\n {k}: {v}"
+ raise ValueError(err_msg)
diff --cc dspy/predict/predict.py
index 88a74861a,a029a1171..259f3a3e9
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@@ -1,17 -1,15 +1,18 @@@
+from __future__ import annotations # Used for self type hints
++from functools import lru_cache
+ import logging
import random
-from functools import lru_cache
+from typing import Optional
from pydantic import BaseModel
import dsp
from dspy.predict.parameter import Parameter
+from dspy.primitives.program import Module
from dspy.primitives.prediction import Prediction
+ from dspy.primitives.program import Module
from dspy.signatures.signature import ensure_signature, signature_to_template
- import logging
- from functools import lru_cache
@lru_cache(maxsize=None)
def warn_once(msg: str):
diff --cc dspy/primitives/module.py
index 4fe9ea692,1dee9cbb1..87ce8b1aa
--- a/dspy/primitives/module.py
+++ b/dspy/primitives/module.py
@@@ -149,11 -149,12 +149,11 @@@ class BaseModule
return new_instance
def dump_state(self, save_verbose):
- print(self.named_parameters())
return {name: param.dump_state(save_verbose) for name, param in self.named_parameters()}
- def load_state(self, state):
+ def load_state(self, state, use_legacy_loading=False):
for name, param in self.named_parameters():
- param.load_state(state[name])
+ param.load_state(state[name], use_legacy_loading=use_legacy_loading)
def save(self, path, save_field_meta=False):
with open(path, "w") as f:
diff --cc dspy/teleprompt/finetune_teleprompter.py
index 36544b34d,000000000..d5c5f563a
mode 100644,000000..100644
--- a/dspy/teleprompt/finetune_teleprompter.py
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@@ -1,415 -1,0 +1,415 @@@
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import dspy
- from dspy import logger
++from dspy.utils.logging import logger
+from dspy.evaluate.evaluate import Evaluate
+from dspy.primitives.example import Example
+from dspy.primitives.program import Program
+from dspy.primitives.prediction import Prediction
+from dspy.signatures.signature import signature_to_template
+
+
+#-------------------------------------------------------------------------------
+# Templates for the user-facing strings used by this module
+#-------------------------------------------------------------------------------
+
+_INFO_DEFAULT_TEACHER = """No teacher provided. Using a copy of the student \
+program as the teacher."""
+
+_INFO_STRUCTURAL_EQUIVALENCY = """Ensuring that the student and teacher are \
+are structurally equivalent."""
+
+_INFO_SHARED_PREDICTOR = """Ensuring that the student and teacher programs do \
+not share predictors."""
+
+_INFO_LM_CONSISTENCY = """Ensuring that the teacher program satisfies the LM \
+consistency property."""
+
+_INFO_BOOTSTRAP_DATA = """Bootstrapping data on {} examples with the program \
+{}, with {} threads"""
+
+#-------------------------------------------------------------------------------
+# Helper functions
+#-------------------------------------------------------------------------------
+
+# TODO: Shared below are useful functions. Similar procedures are implemented
+# separately and used by other DSPy teleprompters. These can be moved to shared
+# locations.
+def prepare_teacher(
+ student: Program,
+ teacher: Program = None
+ ) -> Union[Program, AssertionError]:
+ """Prepare the teacher program with respect to the student program.
+
+ Args:
+ student: The student program.
+ teacher: The teacher program. If `None`, a copy of the student program
+ is used as the teacher. Defaults to `None`.
+
+ Returns:
+ The copied teacher program.
+
+ Raises:
+ AssertionError: If the teacher is not an instance of the Program class.
+ """
+ # If teacher is None, use a copy of the student program as the teacher
+ if teacher is None:
+ logger.info(_INFO_DEFAULT_TEACHER)
+ teacher = student.deepcopy()
+ else:
+ teacher = teacher.deepcopy()
+
+ # Ensure that the student and teacher programs have the same structure
+ logger.info(_INFO_STRUCTURAL_EQUIVALENCY)
+ student._assert_structural_equivalency(teacher)
+
+ # Ensure that the predictors of the programs point to different objects
+ logger.info(_INFO_SHARED_PREDICTOR)
+ student._assert_no_shared_predictor(teacher)
+
+ # Ensure that the LM consistency property is satisfied
+ logger.info(_INFO_LM_CONSISTENCY)
+ teacher._assert_lm_consistency()
+
+ # If the global LM is being used, set it to the LMs of the copied teacher
+ # program predictors to to avoid handling the same edge cases later
+ if dspy.settings.lm:
+ teacher._set_all_predictor_lms(dspy.settings.lm)
+
+ return teacher
+
+
+def build_prompt_completion_data_from_trace(
+ trace: List[Dict],
+ exclude_demos: bool=False,
+ try_to_record_lm_kwargs: bool = False,
+ program: Program = None,
+ ) -> Dict[str, List[Dict[str, Any]]]:
+ """Build prompt completion data from a given trace.
+
+ Args:
+ trace: The trace from which the prompt-completion data will be built.
+ exclude_demos: Exclude the demos from the prompts even if they are
+ present in the trace. Defaults to `False`.
+ try_to_record_lm_kwargs: Whether to record the LM kwargs in the data.
+ Defaults to `False`. If set, the `lm_kwargs` field of the LM used to
+ generate the prompt-completion pair is included in the data. To
+ find the LM, we first check if the predictor that generated the
+ prompt-completion pair has an LM field set (`predictor.lm`). If it
+ does, we record it's kwargs. If it doesn't, we get the kwargs from
+ `dspy.settings.lm`. If `dspy.settings.lm` is not set either, this
+ function will not record the LM kwargs.
+ program: Optional argument used to infer the name of the predictor that
+ generated the prompt-completion pair. If provided, the returned data
+ will include the `predictor_name` field. If not provided, the
+ `predictor_name` field is not included in the data, but the caller
+ of this function can recover this information by using the
+ `predictor_ind` field that' included in the data by default, by
+ building the following dictionary:
+
+ {ind: n for ind, (n, _) in enumerate(program.named_predictors())}
+
+ where `ind` is the index of the predictor in the list returned by
+ the `named_predictors()` method of the program. Defaults to `None`.
+
+ Returns:
+ Data as a list of dictionaries with the keys `prompt`, `completion` and
+ optionally with the keys `predictor_name` and `lm_kwargs`. For a given
+ prompt-completion pair:
+ - The `prompt` field corresponds to the prompt.
+ - The `completion` field corresponds to the completion.
+ - The `predictor_ind` field corresponds to the index of the predictor
+ in the predictor list
+ - The `predictor_name` field corresponds to the index of the predictor
+ in the list returned by the the named_predictor() method of the
+ program used to generate the trace. This field is included only if
+ the `pred_ind_to_name` argument is provided.
+ - The `lm_kwargs` field corresponds to the LM kwargs that generated the
+ prompt-completion pair. Included only if the `record_lm_kwargs` is
+ set and there is an active LM.
+ """
+ # If the program is provided, build the predictor index to name mapping
+ if program:
+ pred_ind_to_name = {
+ ind: name for ind, (name, _) in enumerate(program.named_predictors())
+ }
+
+ # Build the prompt-completion data
+ data = []
+ for pred_ind, (pred, inputs, outputs) in enumerate(trace):
+ # Get the demos from the predictor if exclude_demos is False
+ demos = [] if exclude_demos else pred.demos
+
+ # Build prompt and completion strings
+ template = signature_to_template(pred.signature)
+ prompt = template(Example(demos=demos, **inputs))
+ completion = template.query(Example(**outputs))
+
+ # TODO: This part of the code could be improved.
+ # The method we use to build the completion (template.query) is meant to
+ # be used for creating, well, queries, and hence contains field prefixes
+ # (e.g. "Reasoning: Let's think step by step in order to"), which are
+ # also contained in the last piece of the prompt (separated with a new
+ # line) We remove this piece from the completion. This is a hacky
+ # solution since it assumes a particular template format.
+ prompt_last = prompt.split("\n")[-1]
+ completion = completion[len(prompt_last):]
+
+ # Create prompt-completion dictionary and add it to the data; optionally
+ # add the predictor_name key as well as the lm_kwargs.
+ data_dict = dict(prompt=prompt, completion=completion)
+
+ # Record the predictor index and optionally, name
+ data_dict['predictor_ind'] = pred_ind
+ if program:
+ data_dict['predictor_name'] = pred_ind_to_name[pred_ind]
+
+ # Optionally, record the LM kwargs
+ lm = pred.lm or dspy.settings.lm
+ if try_to_record_lm_kwargs and lm:
+ data_dict['lm_kwargs'] = lm.kwargs
+ data.append(data_dict)
+
+ return data
+
+
+def convert_to_module_level_prompt_completion_data(
+ data: List[Dict],
+ keep_data_keys: bool = False,
+ exclude_demos: bool = False,
+ try_to_record_lm_kwargs: bool = False,
+ program: Program = None
+ ) -> List[Dict]:
+ """Convert the data to prompt-completion data using the "trace" field.
+
+ This function is a wrapper around the function
+ `build_prompt_completion_data_from_trace`, calling it on the "trace" field
+ of each dictionary in the input data list and combiningin the results into
+ a list of prompt-completion data dictionaries. If the `keep_data_keys`
+ is set, the original data keys are also copied over to in the
+ prompt-completion dictionaries.
+
+ For example, if the input data includes 10 dictionaries, each containing a
+ "trace" field generated by a program with 3 predictors, the returned data
+ will have 30 prompt-completion data dictionaries.
+
+ Args:
+ data: List of data dictionaries to be converted to prompt-completion
+ data. Each dictionary in the list should contain a "trace" field,
+ which is passed to the `build_prompt_completion_data_from_trace`
+ function to generate the prompt-completion data.
+ keep_data_keys: Whether to keep the original data keys in the
+ prompt-completion data. Note that if there are keys that are common
+ between the original data and the prompt-completion data returned by
+ `build_prompt_completion_data_from_trace`, the values from the
+ prompt-completion data will overwrite the values from the original
+ data. Defaults to `False`.
+ exclude_demos: Passed to `build_prompt_completion_data_from_trace`.
+ Defaults to `False`.
+ try_to_record_lm_kwargs: Passed to
+ `build_prompt_completion_data_from_trace`. Defaults to `False`.
+ program: Passed to `build_prompt_completion_data_from_trace`.
+ Defaults to `None`.
+ """
+ prompt_completion_data = []
+ for data_dict in data:
+ trace = data_dict["trace"]
+ trace_prompt_comletion_data = build_prompt_completion_data_from_trace(
+ trace=trace, exclude_demos=exclude_demos,
+ try_to_record_lm_kwargs=try_to_record_lm_kwargs, program=program
+ )
+ for prompt_completion_dict in trace_prompt_comletion_data:
+ if keep_data_keys:
+ prompt_completion_dict = {**data_dict, **prompt_completion_dict}
+ prompt_completion_data.append(prompt_completion_dict)
+ return prompt_completion_data
+
+
+def bootstrap_data(
+ program: Program,
+ dataset: List[Example],
+ metric: Optional[Callable[
+ [Example, Prediction, Optional[List]], Union[bool, int, float]
+ ]] = None,
+ num_threads = 1,
+ ) -> List[Dict[str, Any]]:
+ """Bootstrap prediction and trace data for the program using the dataset.
+
+ Args:
+ program: The program that will be used to generate the traces for data
+ collection.
+ dataset: The dataset to be used for data collection.
+ metric: The optional metric to be used to get a score for the example,
+ recorded in a `score` field in the data. If the metric is not
+ provided, the `score` field is not included in the data. Defaults
+ to `None`.
+ num_threads: The number of threads to be used for data collection.
+ Defaults to `1`.
+
+ Returns:
+ Data as a list of dictionaries with the keys `example`, `prediction`,
+ `trace`, and optionally, `score` fields. For a given example:
+ - The `example` field corresponds to the example itself.
+ - The `example_ind` field corresponds to the index of the example in the
+ `dataset`.
+ - The `prediction` field corresponds to the prediction made by the
+ program on the example.
+ - The `trace` field corresponds to the trace generated by the program
+ on the example.
+ - The `score` field corresponds to the metric score of the example, if
+ the metric is provided. Otherwise, it is not included in the data.
+ """
+ data = []
+
+ # Use Evaluate to call the program have the responses cached
+ cname = program.__class__.__name__
+ info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)
+ logger.info(info)
+ evaluator = Evaluate(
+ devset=dataset, num_threads=num_threads, display_progress=True
+ )
+ evaluator(program, metric=metric)
+
+ # Re-iterate over the dataset to build the cached prompt-completion data
+ for example_ind, example in enumerate(dataset):
+
+ # Run the program on the example
+ with dspy.context(trace=[]):
+ prediction = program(**example.inputs())
+ trace = dspy.settings.trace
+ score = metric(example, prediction, trace) if metric else None
+
+ # Build the data dictionary and extend the data list
+ data_dict = dict(example=example, prediction=prediction, trace=trace)
+ data_dict['example_ind'] = example_ind
+ if metric:
+ data_dict['score'] = score
+ data.append(data_dict)
+
+ return data
+
+
+# TODO: If we can ensure to pass the "round" information every time a call is
+# issued to an LM, we can make repetitive un-cached calls to the same LM without
+# modifying it's temperature. This function can be removed then.
+def bootstrap_data_for_round(
+ program: Program,
+ dataset: List[Example],
+ metric: Optional[Callable[
+ [Example, Prediction, Optional[List]], Union[bool, int, float]
+ ]] = None,
+ num_threads = 1,
+ sampling_round: int = 0,
+ sampling_temperature: Optional[float] = 0.9,
+ sampling_temperature_delta: float = 0.001,
+ ) -> Union[List[Dict], AssertionError]:
+ """ Bootstrap data for the given sampling round.
+
+ This is a wrapper function around the `bootstrap_data` function that allows
+ for collecting data for the given `sampling_round`. Due to the way caching
+ works, one cannot get different completions for the same prompt just by
+ querying an LM again, despite using a high sampling temperature. This
+ function is a workaround to get different completions for the same prompt
+ by modifying the sampling temperature of the LM for the specified
+ `sampling_round`. The temperature of an LM is set to the following value
+ for the given `sampling_round`:
+
+ sampling_temperature + sampling_temperature_delta * sampling_round
+
+ If a `sampling_temperature` of `None` is passed, the already set temperature
+ of the LM is used as the sampling temperature instead.
+
+ To sample different completions for the same prompt, this sampling can be
+ called multiple times with different `sampling_round` values. For example:
+ ```
+ num_rounds = 5
+ data = []
+ for round in range(num_rounds):
+ data += bootstrap_data_for_round(
+ program, dataset, metric=metric, sampling_round=round
+ )
+ # Any dataset filtering for the next round can be done here, if needed
+ ```
+
+ The dataset filtering can become a powerful tool. For example, it can be
+ used to filter out the examples that have already had enough high scoring
+ completions generated for them. Here is an illustration:
+ ```
+ num_correct_target = 3
+ correct_score = 1
+ num_rounds = 5
+ data = []
+ for round in range(num_rounds):
+ data += bootstrap_data_for_round(
+ program, dataset, metric=metric, sampling_round=round
+ )
+ correct_bootstraps = [d for d in data if d['score'] == correct_score]
+ correct_counts = Counter([d['example'] for d in correct_bootstraps])
+ dataset = [d for d in dataset if correct_counts[d] < num_correct_target]
+ ```
+
+ Args:
+ program: The program that will be used to generate the traces for data
+ collection.
+ dataset: The dataset to be used for data collection.
+ metric: The optional metric to be used to get a score for the example,
+ recorded in a `score` field in the data. If the metric is not
+ provided, the `score` field is not included in the data. Defaults to
+ `None`.
+ num_threads: The number of threads to be used for data collection.
+ Defaults to `1`.
+ sampling_round: The round index for which the data is being collected.
+ Defaults to `0`.
+ sampling_temperature: The sampling temperature to be used for round
+ `0`. If a value of `None` is passed, the temperature of the LM is
+ used as the sampling temperature instead. Defaults to a high
+ temperature of `0.9`.
+ sampling_temperature_delta: The small temperature delta that's added to
+ the sampling temperature every increment of the round index to
+ generate different completions for the same prompt. Defaults to
+ `0.001`.
+
+ Returns:
+ Data as a list of dictionaries with the keys returned by
+ the `bootstrap_data` function, descriptions for which are shared in the
+ function documentation. This function adds the following extra field
+ to the dictionaries:
+ - The `round` field corresponds to the `sampling_round` argument.
+ """
+ # TODO: [DSPy 2.5] Migrate to using "seed" for sampling
+ # Helper function to adjust the temperature of the LM. If a None temperature
+ # is passed, keep the LM's temperature as is as the base temperature, then
+ # adjust the temperature for the given round.
+ def copy_model_with_updated_temp(lm):
+ temp = sampling_temperature
+ temp = lm.kwargs["temperature"] if temp is None else temp
+ temp = temp + sampling_temperature_delta * sampling_round
+ return lm.copy(temperature=temp)
+
+ # Ensure that the LM consistency is satisfied, which ensures that either (1)
+ # the global LM is set or (2) all the predictors have an LM set.
+ program._assert_lm_consistency()
+
+ # Deepcopy the program and copy the dataset to avoid modifying the original
+ program = program.deepcopy()
+ dataset = dataset.copy()
+
+ # Update the temperature of the LM for the given round
+ context_lm = None
+ if dspy.settings.lm:
+ context_lm = copy_model_with_updated_temp(dspy.settings.lm)
+ else:
+ for pred in program.predictors():
+ pred.lm = copy_model_with_updated_temp(pred.lm)
+
+ # Collect the data for the given round
+ with dspy.context(lm=context_lm):
+ data = bootstrap_data(
+ program, dataset, metric=metric, num_threads=num_threads
+ )
+
+ # Add the round information to the data
+ for data_dict in data:
+ data_dict["round"] = sampling_round
+
+ return data