Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Address comments

dilaradfe9ee8e1428
diff --git a/dspy/clients/anyscale.py b/dspy/clients/anyscale.py
index d30f48486..8c056e7b0 100644
--- a/dspy/clients/anyscale.py
+++ b/dspy/clients/anyscale.py
@@ -12,13 +12,13 @@ from dspy.clients.finetune import (
from dspy.clients.openai import openai_data_validation, check_message_lengths
try:
- # Importing the AnyScale library for users in the AnyScale workspace, where
- # the library is already installed.
- from anyscale.job import JobConfig
+ # AnyScale fine-tuning requires the following additional imports
import anyscale
+ from anyscale.job import JobConfig
except ImportError:
anyscale = None
+
# List of training methods supported by AnyScale
TRAINING_METHODS_ANYSCALE = [
TrainingMethod.SFT,
@@ -26,16 +26,6 @@ TRAINING_METHODS_ANYSCALE = [
PROVIDER_ANYSCALE = "anyscale"
-def anyscale_model_launch(model: str, launch_kwargs: Dict[str, Any]):
- """Launch an AnyScale model."""
- # TODO: Hardcode resources for launching a select server through docker
- raise NotImplementedError("Method `anyscale_model_launch` is not implemented.")
-
-
-def anyscale_model_kill(model: str, launch_kwargs: Dict[str, Any]):
- # TODO: Hardcode resources for killing a select server through docker
- """Kill an AnyScale model."""
- raise NotImplementedError("Method `anyscale_model_kill` is not implemented.")
def is_anyscale_model(model: str) -> bool:
"""Check if the model is an AnyScale model."""
@@ -51,24 +41,13 @@ class FinetuneJobAnyScale(FinetuneJob):
self.model_names = None
super().__init__(*args, **kwargs)
- def cancel(self):
- """Cancel the finetune job."""
- raise NotImplementedError("Method `cancel` is not implemented.")
- # Call the super's cancel method after the custom cancellation logic,
- # so that the future can be cancelled
- # super().cancel()
-
- def status(self):
- """Get the status of the finetune job."""
- raise NotImplementedError("Method `status` is not implemented.")
-
def finetune_anyscale(
job: FinetuneJobAnyScale,
model: str,
- train_method: TrainingMethod,
train_data: List[Dict[str, Any]],
train_kwargs: Optional[Dict[str, Any]]=None,
+ train_method: TrainingMethod = TrainingMethod.SFT,
) -> str:
"""Start the finetune job."""
train_kwargs = train_kwargs or {}
@@ -157,8 +136,6 @@ def verify_dataset(dataset: List[dict[str, Any]]) -> bool:
logger.error(f"Dataset validation failed: {dataset_validation}")
return False
- # TODO: Is this still useful?
- convo_lens = check_message_lengths(dataset)
return True
diff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.py
index 0a8bdcb2c..fc9838ba1 100644
--- a/dspy/clients/finetune.py
+++ b/dspy/clients/finetune.py
@@ -9,8 +9,6 @@ import ujson
from datasets.fingerprint import Hasher
-# TODO: Move to a centralized location with all the other environment variables
-# Set the directory to save the fine-tuned models
def get_finetune_directory() -> str:
"""Get the directory to save the fine-tuned models."""
# TODO: Move to a centralized location with all the other env variables
@@ -57,14 +55,14 @@ class FinetuneJob(Future):
model: str,
train_data: List[Dict[str, Any]],
train_kwargs: Optional[Dict[str, Any]]=None,
- provider: str = "openai",
train_method: TrainingMethod = TrainingMethod.SFT,
+ provider: str = "openai",
):
self.model = model
self.train_data = train_data
self.train_kwargs: Dict[str, Any] = train_kwargs or {}
- self.provider = provider
self.train_method = train_method
+ self.provider = provider
super().__init__()
def get_kwargs(self):
@@ -72,13 +70,10 @@ class FinetuneJob(Future):
model=self.model,
train_data=self.train_data,
train_kwargs=self.train_kwargs,
- provider=self.provider,
train_method=self.train_method,
+ provider=self.provider,
)
- def __str__(self):
- return f"FinetuningJob({self.to_dict()})"
-
def __repr__(self):
return str(self)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 25757109e..937fb8ffa 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -3,16 +3,15 @@ from datetime import datetime
import functools
import os
from pathlib import Path
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Dict, List, Optional
import ujson
import uuid
from dspy.utils.logging import logger
from dspy.clients.finetune import FinetuneJob, TrainingMethod
-from dspy.clients.anyscale import (
- is_anyscale_model,
- anyscale_model_launch,
- anyscale_model_kill,
+from dspy.clients.lm_finetune_utils import (
+ get_provider_finetune_job_class,
+ execute_finetune_job,
)
import litellm
@@ -28,7 +27,8 @@ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
class LM:
- def __init__(self,
+ def __init__(
+ self,
model,
model_type='chat',
temperature=0.0,
@@ -87,48 +87,39 @@ class LM:
def launch(self):
"""Send a request to the provider to launch the model, if needed."""
- if is_self_hosted_model(self.model):
- self_hosted_model_launch(self.model, self.launch_kwargs)
- elif is_anyscale_model(self.model):
- anyscale_model_launch(self.model, self.launch_kwargs)
msg = f"`launch()` is called for the auto-launched model {self.model}"
msg += " -- no action is taken!"
logger.info(msg)
def kill(self):
"""Send a request to the provider to kill the model, if needed."""
- if is_self_hosted_model(self.model):
- self_hosted_model_kill(self.model, self.launch_kwargs)
- elif is_anyscale_model(self.model):
- anyscale_model_kill(self.model, self.launch_kwargs)
msg = f"`kill()` is called for the auto-launched model {self.model}"
msg += " -- no action is taken!"
logger.info(msg)
- def finetune(self,
- train_method: TrainingMethod,
+ def finetune(
+ self,
train_data: List[Dict[str, Any]],
train_kwargs: Optional[Dict[str, Any]]=None,
- cache_finetune: bool = True,
+ train_method: TrainingMethod = TrainingMethod.SFT,
provider: str = "openai",
+ cache_finetune: bool = True,
) -> FinetuneJob:
"""Start model fine-tuning, if supported."""
- # Fine-tuning is experimental and requires the experimental flag
from dspy import settings as settings
err = "Fine-tuning is an experimental feature."
err += " Set `dspy.settings.experimental` to `True` to use it."
assert settings.experimental, err
- # Initialize the finetune job
FinetuneJobClass = get_provider_finetune_job_class(provider=provider)
finetune_job = FinetuneJobClass(
model=self.model,
- provider=provider,
train_data=train_data,
train_kwargs=train_kwargs,
+ train_method=train_method,
+ provider=provider
)
- # Start asyncronous training
executor = ThreadPoolExecutor(max_workers=1)
executor.submit(
execute_finetune_job,
@@ -229,114 +220,3 @@ def _inspect_history(lm, n: int = 1):
print(_red(choices_text, end=""))
print("\n\n\n")
-
-# TODO: This part can be moved to a separate module
-
-from dspy.clients.openai import (
- FinetuneJobOpenAI,
- finetune_openai,
-)
-from dspy.clients.anyscale import (
- FinetuneJobAnyScale,
- finetune_anyscale,
-)
-
-
-_PROVIDER_ANYSCALE = "anyscale"
-_PROVIDER_OPENAI = "openai"
-
-
-def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:
- """Get the FinetuneJob class for the provider."""
- # Mapping from provider to finetune job type
- _PROVIDER_TO_FINETUNE_JOB_CLASS = {
- _PROVIDER_ANYSCALE: FinetuneJobAnyScale,
- _PROVIDER_OPENAI: FinetuneJobOpenAI,
- }
-
- # Get the FinetuneJob class for the provider
- _CLS = _PROVIDER_TO_FINETUNE_JOB_CLASS[provider]
-
- return _CLS
-
-
-def get_provider_finetune_function(provider: str) -> callable:
- """Return the finetune function for the given model."""
- # Mapping from provider to finetune function
- _PROVIDER_TO_FINETUNE_FUNCTION = {
- _PROVIDER_ANYSCALE: finetune_anyscale,
- _PROVIDER_OPENAI: finetune_openai,
- }
-
- # Get the finetuning provider
- finetune_function = _PROVIDER_TO_FINETUNE_FUNCTION[provider]
-
- return finetune_function
-
-
-def execute_finetune_job(
- job: FinetuneJob[Type[LM]],
- lm: LM,
- cache_finetune: bool=True
-):
- """Execute the finetune job in a blocking manner."""
- try:
- job_kwargs = job.get_kwargs()
- if cache_finetune:
- model = cached_finetune(job=job, **job_kwargs)
- else:
- model = finetune(job=job, **job_kwargs)
- lm = lm.copy(model=model)
- job.set_result(lm)
- except Exception as err:
- logger.error(err)
- job.set_result(err)
-
-
-# TODO: Perhaps we shouldn't directly cache the data
-# TODO: Add DiskCache, ignore job
-
-def cached_finetune(
- job,
- model: str,
- train_data: List[Dict[str, Any]],
- train_kwargs: Optional[Dict[str, Any]]=None,
- provider: str = "openai",
- train_method: TrainingMethod = TrainingMethod.SFT,
-) -> Union[str, ValueError]:
- return finetune(
- job=job,
- model=model,
- provider=provider,
- train_data=train_data,
- train_kwargs=train_kwargs,
- train_method=train_method,
- )
-
-
-def finetune(
- job,
- model: str,
- train_method: TrainingMethod,
- provider: str,
- train_data: List[Dict[str, Any]],
- train_kwargs: Optional[Dict[str, Any]]=None,
-) -> Union[str, Exception]:
- """Fine-tune a new model based on the given model."""
- # Get the fine-tuning provider
- try:
- # Get the finetune function
- provider_finetune_function = get_provider_finetune_function(provider)
-
- # Fine-tune a new model based on the given model
- model = provider_finetune_function(
- job=job,
- model=model,
- train_method=train_method,
- train_data=train_data,
- train_kwargs=train_kwargs,
- )
- except Exception as err:
- raise err
-
- return model
diff --git a/dspy/clients/lm_finetune_utils.py b/dspy/clients/lm_finetune_utils.py
new file mode 100644
index 000000000..6efdf5622
--- /dev/null
+++ b/dspy/clients/lm_finetune_utils.py
@@ -0,0 +1,96 @@
+from typing import Any, Dict, List, Optional, Type, Union
+
+from dspy.utils.logging import logger
+from dspy.clients.finetune import FinetuneJob, TrainingMethod
+from dspy.clients.openai import FinetuneJobOpenAI, finetune_openai
+from dspy.clients.anyscale import FinetuneJobAnyScale, finetune_anyscale
+
+
+_PROVIDER_ANYSCALE = "anyscale"
+_PROVIDER_OPENAI = "openai"
+
+
+def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:
+ """Get the FinetuneJob class for the provider."""
+ provider_to_job_class = {
+ _PROVIDER_ANYSCALE: FinetuneJobAnyScale,
+ _PROVIDER_OPENAI: FinetuneJobOpenAI,
+ }
+ return provider_to_job_class[provider]
+
+
+def get_provider_finetune_function(provider: str) -> callable:
+ """Return the finetune function for the given model."""
+ provider_to_finetune_function = {
+ _PROVIDER_ANYSCALE: finetune_anyscale,
+ _PROVIDER_OPENAI: finetune_openai,
+ }
+ return provider_to_finetune_function[provider]
+
+
+# Note: Type of LM should be LM. We aren't importing it here to avoid
+# circular imports.
+def execute_finetune_job(
+ job: FinetuneJob,
+ lm: Any,
+ cache_finetune: bool=True
+):
+ """Execute the finetune job in a blocking manner."""
+ try:
+ job_kwargs = job.get_kwargs()
+ if cache_finetune:
+ model = cached_finetune(job=job, **job_kwargs)
+ else:
+ model = finetune(job=job, **job_kwargs)
+ lm = lm.copy(model=model)
+ job.set_result(lm)
+ except Exception as err:
+ logger.error(err)
+ job.set_result(err)
+
+
+# TODO: Add DiskCache, ignore job
+def cached_finetune(
+ job,
+ model: str,
+ train_data: List[Dict[str, Any]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+ train_method: TrainingMethod = TrainingMethod.SFT,
+ provider: str = "openai",
+) -> Union[str, Exception]:
+ return finetune(
+ job=job,
+ model=model,
+ train_data=train_data,
+ train_kwargs=train_kwargs,
+ train_method=train_method,
+ provider=provider,
+ )
+
+
+def finetune(
+ job,
+ model: str,
+ train_data: List[Dict[str, Any]],
+ train_kwargs: Optional[Dict[str, Any]]=None,
+ train_method: TrainingMethod = TrainingMethod.SFT,
+ provider: str = "openai",
+) -> Union[str, Exception]:
+ """Fine-tune a new model based on the given model."""
+ # Get the fine-tuning provider
+ try:
+ # Get the finetune function
+ provider_finetune_function = get_provider_finetune_function(provider)
+
+ # Fine-tune a new model based on the given model
+ model = provider_finetune_function(
+ job=job,
+ model=model,
+ train_data=train_data,
+ train_kwargs=train_kwargs,
+ train_method=train_method,
+ )
+ except Exception as err:
+ raise err
+
+ return model
diff --git a/dspy/clients/openai.py b/dspy/clients/openai.py
index 03e64f7c7..de6d66cef 100644
--- a/dspy/clients/openai.py
+++ b/dspy/clients/openai.py
@@ -82,8 +82,6 @@ def is_openai_model(model: str) -> bool:
if match and match.group(1) in _MODEL_IDS:
return True
- # If the model is not a base OpenAI model or a fine-tuned OpenAI model, then
- # it is not an OpenAI model.
return False
class FinetuneJobOpenAI(FinetuneJob):
@@ -122,9 +120,9 @@ class FinetuneJobOpenAI(FinetuneJob):
def finetune_openai(
job: FinetuneJobOpenAI,
model: str,
- method: TrainingMethod,
train_data: List[Dict[str, Any]],
train_kwargs: Optional[Dict[str, Any]]=None,
+ train_method: TrainingMethod = TrainingMethod.SFT,
) -> str:
train_kwargs = train_kwargs or {}
train_method = TrainingMethod.SFT # Note: This could be an argument; ignoring method
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 96034cee9..31ad3e09b 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -6,8 +6,8 @@ from pydantic import BaseModel
import dsp
from dspy.predict.parameter import Parameter
-from dspy.primitives.program import Module
from dspy.primitives.prediction import Prediction
+from dspy.primitives.program import Module
from dspy.signatures.signature import ensure_signature, signature_to_template
diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.py
index 1001954d3..360461bd8 100644
--- a/dspy/primitives/__init__.py
+++ b/dspy/primitives/__init__.py
@@ -1,5 +1,4 @@
from .assertions import *
from .example import *
from .prediction import *
-from .program import *
-from .python_interpreter import *
\ No newline at end of file
+from .program import *
\ No newline at end of file
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index ce6322b58..51eed2d87 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -1,4 +1,3 @@
-
import magicattr
from dsp.modules.lm import LM
diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.py
index 7a8c519ec..cb170958c 100644
--- a/dspy/teleprompt/finetune_teleprompter.py
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@ -7,20 +7,6 @@ from dspy.primitives.example import Example
from dspy.primitives.program import Program
from dspy.primitives.prediction import Prediction
-_INFO_DEFAULT_TEACHER = """No teacher provided. Using a copy of the student \
-program as the teacher."""
-
-_INFO_STRUCTURAL_EQUIVALENCY = """Ensuring that the student and teacher are \
-are structurally equivalent."""
-
-_INFO_SHARED_PREDICTOR = """Ensuring that the student and teacher programs do \
-not share predictors."""
-
-_INFO_LM_CONSISTENCY = """Ensuring that the teacher program satisfies the LM \
-consistency property."""
-
-_INFO_BOOTSTRAP_DATA = """Bootstrapping data on {} examples with the program \
-{}, with {} threads"""
# TODO: Shared below are useful functions. Similar procedures are implemented
# separately and used by other DSPy teleprompters. These can be moved to shared
@@ -37,21 +23,21 @@ def prepare_teacher(
"""
# If teacher is None, use a copy of the student program as the teacher
if teacher is None:
- logger.info(_INFO_DEFAULT_TEACHER)
+ logger.info("No teacher provided. Using a copy of the student program as the teacher.")
teacher = student.deepcopy()
else:
teacher = teacher.deepcopy()
# Ensure that the student and teacher programs have the same structure
- logger.info(_INFO_STRUCTURAL_EQUIVALENCY)
+ logger.info("Ensuring that the student and teacher are are structurally equivalent.")
student._assert_structural_equivalency(teacher)
# Ensure that the predictors of the programs point to different objects
- logger.info(_INFO_SHARED_PREDICTOR)
+ logger.info("Ensuring that the student and teacher programs do not share predictors.")
student._assert_no_shared_predictor(teacher)
# Ensure that the LM consistency property is satisfied
- logger.info(_INFO_LM_CONSISTENCY)
+ logger.info("Ensuring that the teacher program satisfies the LM consistency property.")
teacher._assert_lm_consistency()
# If the global LM is being used, set it to the LMs of the copied teacher
@@ -62,7 +48,6 @@ def prepare_teacher(
return teacher
-# TODO: fix docstring
def convert_to_module_level_message_data(
data: List[Dict],
keep_data_keys: bool = False,
@@ -88,6 +73,7 @@ def convert_to_module_level_message_data(
prompt_completion_data.append(prompt_completion_dict)
return prompt_completion_data
+
def build_messages_from_trace(
trace: List[Dict],
exclude_demos: bool=False,
@@ -116,15 +102,13 @@ def build_messages_from_trace(
return data
-def dummy_metric(example, pred, trace=None, frac=1.0):
- return 1
def bootstrap_data(
program: Program,
dataset: List[Example],
metric: Optional[Callable[
[Example, Prediction, Optional[List]], Union[bool, int, float]
- ]] = dummy_metric,
+ ]] = None,
num_threads = 1,
max_errors: int = 0
) -> List[Dict[str, Any]]:
@@ -133,7 +117,7 @@ def bootstrap_data(
# Use Evaluate to call the program have the responses cached
cname = program.__class__.__name__
- info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)
+ info = "Bootstrapping data on {} examples with the program {}, with {} threads".format(len(dataset), cname, num_threads)
logger.info(info)
evaluator = Evaluate(
devset=dataset, num_threads=num_threads, display_progress=True, max_errors=max_errors, provide_traceback=True
@@ -141,26 +125,28 @@ def bootstrap_data(
evaluator(program, metric=metric)
data = []
- for example in dataset:
- data_dict = process_example(example, 0, program, metric)
+ for example_ind, example in enumerate(dataset):
+ data_dict = bootstrap_one_example(
+ example=example,
+ example_ind=example_ind,
+ program=program,
+ metric=metric
+ )
if data_dict is not None:
data.append(data_dict)
return data
-def process_example(example: Any, example_ind: int, program: Callable, metric: Optional[Callable] = None) -> Dict[str, Any]:
- # print("Processing example:", example_ind)
+def bootstrap_one_example(
+ example: Any,
+ example_ind: int,
+ program: Program,
+ metric: Optional[Callable] = None
+ ) -> Dict[str, Any]:
with dspy.context(trace=[]):
- # print("Running program...", example_ind)
- try:
- prediction = program(**example.inputs())
- except Exception as e:
- print(f"Error processing example {example_ind}: {e}")
- return None
- # print("Getting trace...", example_ind)
+ prediction = program(**example.inputs())
trace = dspy.settings.trace
- # print("Getting score...", example_ind)
score = metric(example, prediction, trace) if metric else None
data_dict = {
@@ -173,65 +159,3 @@ def process_example(example: Any, example_ind: int, program: Callable, metric: O
data_dict['score'] = score
return data_dict
-
-# TODO: If we can ensure to pass the "round" information every time a call is
-# issued to an LM, we can make repetitive un-cached calls to the same LM without
-# modifying it's temperature. This function can be removed then.
-def bootstrap_data_for_round(
- program: Program,
- dataset: List[Example],
- metric: Optional[Callable[
- [Example, Prediction, Optional[List]], Union[bool, int, float]
- ]] = None,
- num_threads = 1,
- sampling_round: int = 0,
- sampling_temperature: Optional[float] = 0.9,
- sampling_temperature_delta: float = 0.001,
- max_errors: int = 0
- ) -> List[Dict]:
- """ Bootstrap data for the given sampling round.
-
- This is a wrapper function around the `bootstrap_data` function that allows
- for collecting data for the given `sampling_round`. Due to the way caching
- works, one cannot get different completions for the same prompt just by
- querying an LM again.
- """
- # TODO: [DSPy 2.5] Migrate to using "seed" for sampling
- # Helper function to adjust the temperature of the LM. If a None temperature
- # is passed, keep the LM's temperature as is as the base temperature, then
- # adjust the temperature for the given round.
- def copy_model_with_updated_temp(lm):
- temp = sampling_temperature
- temp = lm.kwargs["temperature"] if temp is None else temp
- temp = temp + sampling_temperature_delta * sampling_round
- return lm.copy(temperature=temp)
-
- # Ensure that the LM consistency is satisfied, which ensures that either (1)
- # the global LM is set or (2) all the predictors have an LM set.
- # TODO(isaac): Uncomment this line after the LM consistency property is
- # program._assert_lm_consistency()
-
- # Deepcopy the program and copy the dataset to avoid modifying the original
- program = program.deepcopy()
- dataset = dataset.copy()
-
- # Update the temperature of the LM for the given round
- context_lm = None
- if dspy.settings.lm:
- context_lm = copy_model_with_updated_temp(dspy.settings.lm)
- else:
- for pred in program.predictors():
- pred.lm = copy_model_with_updated_temp(pred.lm)
-
- # Collect the data for the given round
- with dspy.context(lm=context_lm):
- # print(context_lm.kwargs)
- data = bootstrap_data(
- program, dataset, metric=metric, num_threads=num_threads, max_errors=max_errors
- )
-
- # Add the round information to the data
- for data_dict in data:
- data_dict["round"] = sampling_round
-
- return data