DSPy: The framework for programming—not prompting—language models
Python
Address comments
dilaradfe9ee8e1428
diff --git a/dspy/clients/anyscale.py b/dspy/clients/anyscale.pyindex d30f48486..8c056e7b0 100644--- a/dspy/clients/anyscale.py+++ b/dspy/clients/anyscale.py@@ -12,13 +12,13 @@ from dspy.clients.finetune import (from dspy.clients.openai import openai_data_validation, check_message_lengthstry:- # Importing the AnyScale library for users in the AnyScale workspace, where- # the library is already installed.- from anyscale.job import JobConfig+ # AnyScale fine-tuning requires the following additional importsimport anyscale+ from anyscale.job import JobConfigexcept ImportError:anyscale = None+# List of training methods supported by AnyScaleTRAINING_METHODS_ANYSCALE = [TrainingMethod.SFT,@@ -26,16 +26,6 @@ TRAINING_METHODS_ANYSCALE = [PROVIDER_ANYSCALE = "anyscale"-def anyscale_model_launch(model: str, launch_kwargs: Dict[str, Any]):- """Launch an AnyScale model."""- # TODO: Hardcode resources for launching a select server through docker- raise NotImplementedError("Method `anyscale_model_launch` is not implemented.")---def anyscale_model_kill(model: str, launch_kwargs: Dict[str, Any]):- # TODO: Hardcode resources for killing a select server through docker- """Kill an AnyScale model."""- raise NotImplementedError("Method `anyscale_model_kill` is not implemented.")def is_anyscale_model(model: str) -> bool:"""Check if the model is an AnyScale model."""@@ -51,24 +41,13 @@ class FinetuneJobAnyScale(FinetuneJob):self.model_names = Nonesuper().__init__(*args, **kwargs)- def cancel(self):- """Cancel the finetune job."""- raise NotImplementedError("Method `cancel` is not implemented.")- # Call the super's cancel method after the custom cancellation logic,- # so that the future can be cancelled- # super().cancel()-- def status(self):- """Get the status of the finetune job."""- raise NotImplementedError("Method `status` is not implemented.")-def finetune_anyscale(job: FinetuneJobAnyScale,model: str,- train_method: TrainingMethod,train_data: List[Dict[str, Any]],train_kwargs: Optional[Dict[str, Any]]=None,+ train_method: TrainingMethod = TrainingMethod.SFT,) -> str:"""Start the finetune job."""train_kwargs = train_kwargs or {}@@ -157,8 +136,6 @@ def verify_dataset(dataset: List[dict[str, Any]]) -> bool:logger.error(f"Dataset validation failed: {dataset_validation}")return False- # TODO: Is this still useful?- convo_lens = check_message_lengths(dataset)return Truediff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.pyindex 0a8bdcb2c..fc9838ba1 100644--- a/dspy/clients/finetune.py+++ b/dspy/clients/finetune.py@@ -9,8 +9,6 @@ import ujsonfrom datasets.fingerprint import Hasher-# TODO: Move to a centralized location with all the other environment variables-# Set the directory to save the fine-tuned modelsdef get_finetune_directory() -> str:"""Get the directory to save the fine-tuned models."""# TODO: Move to a centralized location with all the other env variables@@ -57,14 +55,14 @@ class FinetuneJob(Future):model: str,train_data: List[Dict[str, Any]],train_kwargs: Optional[Dict[str, Any]]=None,- provider: str = "openai",train_method: TrainingMethod = TrainingMethod.SFT,+ provider: str = "openai",):self.model = modelself.train_data = train_dataself.train_kwargs: Dict[str, Any] = train_kwargs or {}- self.provider = providerself.train_method = train_method+ self.provider = providersuper().__init__()def get_kwargs(self):@@ -72,13 +70,10 @@ class FinetuneJob(Future):model=self.model,train_data=self.train_data,train_kwargs=self.train_kwargs,- provider=self.provider,train_method=self.train_method,+ provider=self.provider,)- def __str__(self):- return f"FinetuningJob({self.to_dict()})"-def __repr__(self):return str(self)diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 25757109e..937fb8ffa 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -3,16 +3,15 @@ from datetime import datetimeimport functoolsimport osfrom pathlib import Path-from typing import Any, Dict, List, Optional, Type, Union+from typing import Any, Dict, List, Optionalimport ujsonimport uuidfrom dspy.utils.logging import loggerfrom dspy.clients.finetune import FinetuneJob, TrainingMethod-from dspy.clients.anyscale import (- is_anyscale_model,- anyscale_model_launch,- anyscale_model_kill,+from dspy.clients.lm_finetune_utils import (+ get_provider_finetune_job_class,+ execute_finetune_job,)import litellm@@ -28,7 +27,8 @@ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:class LM:- def __init__(self,+ def __init__(+ self,model,model_type='chat',temperature=0.0,@@ -87,48 +87,39 @@ class LM:def launch(self):"""Send a request to the provider to launch the model, if needed."""- if is_self_hosted_model(self.model):- self_hosted_model_launch(self.model, self.launch_kwargs)- elif is_anyscale_model(self.model):- anyscale_model_launch(self.model, self.launch_kwargs)msg = f"`launch()` is called for the auto-launched model {self.model}"msg += " -- no action is taken!"logger.info(msg)def kill(self):"""Send a request to the provider to kill the model, if needed."""- if is_self_hosted_model(self.model):- self_hosted_model_kill(self.model, self.launch_kwargs)- elif is_anyscale_model(self.model):- anyscale_model_kill(self.model, self.launch_kwargs)msg = f"`kill()` is called for the auto-launched model {self.model}"msg += " -- no action is taken!"logger.info(msg)- def finetune(self,- train_method: TrainingMethod,+ def finetune(+ self,train_data: List[Dict[str, Any]],train_kwargs: Optional[Dict[str, Any]]=None,- cache_finetune: bool = True,+ train_method: TrainingMethod = TrainingMethod.SFT,provider: str = "openai",+ cache_finetune: bool = True,) -> FinetuneJob:"""Start model fine-tuning, if supported."""- # Fine-tuning is experimental and requires the experimental flagfrom dspy import settings as settingserr = "Fine-tuning is an experimental feature."err += " Set `dspy.settings.experimental` to `True` to use it."assert settings.experimental, err- # Initialize the finetune jobFinetuneJobClass = get_provider_finetune_job_class(provider=provider)finetune_job = FinetuneJobClass(model=self.model,- provider=provider,train_data=train_data,train_kwargs=train_kwargs,+ train_method=train_method,+ provider=provider)- # Start asyncronous trainingexecutor = ThreadPoolExecutor(max_workers=1)executor.submit(execute_finetune_job,@@ -229,114 +220,3 @@ def _inspect_history(lm, n: int = 1):print(_red(choices_text, end=""))print("\n\n\n")--# TODO: This part can be moved to a separate module--from dspy.clients.openai import (- FinetuneJobOpenAI,- finetune_openai,-)-from dspy.clients.anyscale import (- FinetuneJobAnyScale,- finetune_anyscale,-)---_PROVIDER_ANYSCALE = "anyscale"-_PROVIDER_OPENAI = "openai"---def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:- """Get the FinetuneJob class for the provider."""- # Mapping from provider to finetune job type- _PROVIDER_TO_FINETUNE_JOB_CLASS = {- _PROVIDER_ANYSCALE: FinetuneJobAnyScale,- _PROVIDER_OPENAI: FinetuneJobOpenAI,- }-- # Get the FinetuneJob class for the provider- _CLS = _PROVIDER_TO_FINETUNE_JOB_CLASS[provider]-- return _CLS---def get_provider_finetune_function(provider: str) -> callable:- """Return the finetune function for the given model."""- # Mapping from provider to finetune function- _PROVIDER_TO_FINETUNE_FUNCTION = {- _PROVIDER_ANYSCALE: finetune_anyscale,- _PROVIDER_OPENAI: finetune_openai,- }-- # Get the finetuning provider- finetune_function = _PROVIDER_TO_FINETUNE_FUNCTION[provider]-- return finetune_function---def execute_finetune_job(- job: FinetuneJob[Type[LM]],- lm: LM,- cache_finetune: bool=True-):- """Execute the finetune job in a blocking manner."""- try:- job_kwargs = job.get_kwargs()- if cache_finetune:- model = cached_finetune(job=job, **job_kwargs)- else:- model = finetune(job=job, **job_kwargs)- lm = lm.copy(model=model)- job.set_result(lm)- except Exception as err:- logger.error(err)- job.set_result(err)---# TODO: Perhaps we shouldn't directly cache the data-# TODO: Add DiskCache, ignore job--def cached_finetune(- job,- model: str,- train_data: List[Dict[str, Any]],- train_kwargs: Optional[Dict[str, Any]]=None,- provider: str = "openai",- train_method: TrainingMethod = TrainingMethod.SFT,-) -> Union[str, ValueError]:- return finetune(- job=job,- model=model,- provider=provider,- train_data=train_data,- train_kwargs=train_kwargs,- train_method=train_method,- )---def finetune(- job,- model: str,- train_method: TrainingMethod,- provider: str,- train_data: List[Dict[str, Any]],- train_kwargs: Optional[Dict[str, Any]]=None,-) -> Union[str, Exception]:- """Fine-tune a new model based on the given model."""- # Get the fine-tuning provider- try:- # Get the finetune function- provider_finetune_function = get_provider_finetune_function(provider)-- # Fine-tune a new model based on the given model- model = provider_finetune_function(- job=job,- model=model,- train_method=train_method,- train_data=train_data,- train_kwargs=train_kwargs,- )- except Exception as err:- raise err-- return modeldiff --git a/dspy/clients/lm_finetune_utils.py b/dspy/clients/lm_finetune_utils.pynew file mode 100644index 000000000..6efdf5622--- /dev/null+++ b/dspy/clients/lm_finetune_utils.py@@ -0,0 +1,96 @@+from typing import Any, Dict, List, Optional, Type, Union++from dspy.utils.logging import logger+from dspy.clients.finetune import FinetuneJob, TrainingMethod+from dspy.clients.openai import FinetuneJobOpenAI, finetune_openai+from dspy.clients.anyscale import FinetuneJobAnyScale, finetune_anyscale+++_PROVIDER_ANYSCALE = "anyscale"+_PROVIDER_OPENAI = "openai"+++def get_provider_finetune_job_class(provider: str) -> Type[FinetuneJob]:+ """Get the FinetuneJob class for the provider."""+ provider_to_job_class = {+ _PROVIDER_ANYSCALE: FinetuneJobAnyScale,+ _PROVIDER_OPENAI: FinetuneJobOpenAI,+ }+ return provider_to_job_class[provider]+++def get_provider_finetune_function(provider: str) -> callable:+ """Return the finetune function for the given model."""+ provider_to_finetune_function = {+ _PROVIDER_ANYSCALE: finetune_anyscale,+ _PROVIDER_OPENAI: finetune_openai,+ }+ return provider_to_finetune_function[provider]+++# Note: Type of LM should be LM. We aren't importing it here to avoid+# circular imports.+def execute_finetune_job(+ job: FinetuneJob,+ lm: Any,+ cache_finetune: bool=True+):+ """Execute the finetune job in a blocking manner."""+ try:+ job_kwargs = job.get_kwargs()+ if cache_finetune:+ model = cached_finetune(job=job, **job_kwargs)+ else:+ model = finetune(job=job, **job_kwargs)+ lm = lm.copy(model=model)+ job.set_result(lm)+ except Exception as err:+ logger.error(err)+ job.set_result(err)+++# TODO: Add DiskCache, ignore job+def cached_finetune(+ job,+ model: str,+ train_data: List[Dict[str, Any]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ train_method: TrainingMethod = TrainingMethod.SFT,+ provider: str = "openai",+) -> Union[str, Exception]:+ return finetune(+ job=job,+ model=model,+ train_data=train_data,+ train_kwargs=train_kwargs,+ train_method=train_method,+ provider=provider,+ )+++def finetune(+ job,+ model: str,+ train_data: List[Dict[str, Any]],+ train_kwargs: Optional[Dict[str, Any]]=None,+ train_method: TrainingMethod = TrainingMethod.SFT,+ provider: str = "openai",+) -> Union[str, Exception]:+ """Fine-tune a new model based on the given model."""+ # Get the fine-tuning provider+ try:+ # Get the finetune function+ provider_finetune_function = get_provider_finetune_function(provider)++ # Fine-tune a new model based on the given model+ model = provider_finetune_function(+ job=job,+ model=model,+ train_data=train_data,+ train_kwargs=train_kwargs,+ train_method=train_method,+ )+ except Exception as err:+ raise err++ return modeldiff --git a/dspy/clients/openai.py b/dspy/clients/openai.pyindex 03e64f7c7..de6d66cef 100644--- a/dspy/clients/openai.py+++ b/dspy/clients/openai.py@@ -82,8 +82,6 @@ def is_openai_model(model: str) -> bool:if match and match.group(1) in _MODEL_IDS:return True- # If the model is not a base OpenAI model or a fine-tuned OpenAI model, then- # it is not an OpenAI model.return Falseclass FinetuneJobOpenAI(FinetuneJob):@@ -122,9 +120,9 @@ class FinetuneJobOpenAI(FinetuneJob):def finetune_openai(job: FinetuneJobOpenAI,model: str,- method: TrainingMethod,train_data: List[Dict[str, Any]],train_kwargs: Optional[Dict[str, Any]]=None,+ train_method: TrainingMethod = TrainingMethod.SFT,) -> str:train_kwargs = train_kwargs or {}train_method = TrainingMethod.SFT # Note: This could be an argument; ignoring methoddiff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 96034cee9..31ad3e09b 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -6,8 +6,8 @@ from pydantic import BaseModelimport dspfrom dspy.predict.parameter import Parameter-from dspy.primitives.program import Modulefrom dspy.primitives.prediction import Prediction+from dspy.primitives.program import Modulefrom dspy.signatures.signature import ensure_signature, signature_to_templatediff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.pyindex 1001954d3..360461bd8 100644--- a/dspy/primitives/__init__.py+++ b/dspy/primitives/__init__.py@@ -1,5 +1,4 @@from .assertions import *from .example import *from .prediction import *-from .program import *-from .python_interpreter import *\ No newline at end of file+from .program import *\ No newline at end of filediff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex ce6322b58..51eed2d87 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -1,4 +1,3 @@-import magicattrfrom dsp.modules.lm import LMdiff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.pyindex 7a8c519ec..cb170958c 100644--- a/dspy/teleprompt/finetune_teleprompter.py+++ b/dspy/teleprompt/finetune_teleprompter.py@@ -7,20 +7,6 @@ from dspy.primitives.example import Examplefrom dspy.primitives.program import Programfrom dspy.primitives.prediction import Prediction-_INFO_DEFAULT_TEACHER = """No teacher provided. Using a copy of the student \-program as the teacher."""--_INFO_STRUCTURAL_EQUIVALENCY = """Ensuring that the student and teacher are \-are structurally equivalent."""--_INFO_SHARED_PREDICTOR = """Ensuring that the student and teacher programs do \-not share predictors."""--_INFO_LM_CONSISTENCY = """Ensuring that the teacher program satisfies the LM \-consistency property."""--_INFO_BOOTSTRAP_DATA = """Bootstrapping data on {} examples with the program \-{}, with {} threads"""# TODO: Shared below are useful functions. Similar procedures are implemented# separately and used by other DSPy teleprompters. These can be moved to shared@@ -37,21 +23,21 @@ def prepare_teacher("""# If teacher is None, use a copy of the student program as the teacherif teacher is None:- logger.info(_INFO_DEFAULT_TEACHER)+ logger.info("No teacher provided. Using a copy of the student program as the teacher.")teacher = student.deepcopy()else:teacher = teacher.deepcopy()# Ensure that the student and teacher programs have the same structure- logger.info(_INFO_STRUCTURAL_EQUIVALENCY)+ logger.info("Ensuring that the student and teacher are are structurally equivalent.")student._assert_structural_equivalency(teacher)# Ensure that the predictors of the programs point to different objects- logger.info(_INFO_SHARED_PREDICTOR)+ logger.info("Ensuring that the student and teacher programs do not share predictors.")student._assert_no_shared_predictor(teacher)# Ensure that the LM consistency property is satisfied- logger.info(_INFO_LM_CONSISTENCY)+ logger.info("Ensuring that the teacher program satisfies the LM consistency property.")teacher._assert_lm_consistency()# If the global LM is being used, set it to the LMs of the copied teacher@@ -62,7 +48,6 @@ def prepare_teacher(return teacher-# TODO: fix docstringdef convert_to_module_level_message_data(data: List[Dict],keep_data_keys: bool = False,@@ -88,6 +73,7 @@ def convert_to_module_level_message_data(prompt_completion_data.append(prompt_completion_dict)return prompt_completion_data+def build_messages_from_trace(trace: List[Dict],exclude_demos: bool=False,@@ -116,15 +102,13 @@ def build_messages_from_trace(return data-def dummy_metric(example, pred, trace=None, frac=1.0):- return 1def bootstrap_data(program: Program,dataset: List[Example],metric: Optional[Callable[[Example, Prediction, Optional[List]], Union[bool, int, float]- ]] = dummy_metric,+ ]] = None,num_threads = 1,max_errors: int = 0) -> List[Dict[str, Any]]:@@ -133,7 +117,7 @@ def bootstrap_data(# Use Evaluate to call the program have the responses cachedcname = program.__class__.__name__- info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)+ info = "Bootstrapping data on {} examples with the program {}, with {} threads".format(len(dataset), cname, num_threads)logger.info(info)evaluator = Evaluate(devset=dataset, num_threads=num_threads, display_progress=True, max_errors=max_errors, provide_traceback=True@@ -141,26 +125,28 @@ def bootstrap_data(evaluator(program, metric=metric)data = []- for example in dataset:- data_dict = process_example(example, 0, program, metric)+ for example_ind, example in enumerate(dataset):+ data_dict = bootstrap_one_example(+ example=example,+ example_ind=example_ind,+ program=program,+ metric=metric+ )if data_dict is not None:data.append(data_dict)return data-def process_example(example: Any, example_ind: int, program: Callable, metric: Optional[Callable] = None) -> Dict[str, Any]:- # print("Processing example:", example_ind)+def bootstrap_one_example(+ example: Any,+ example_ind: int,+ program: Program,+ metric: Optional[Callable] = None+ ) -> Dict[str, Any]:with dspy.context(trace=[]):- # print("Running program...", example_ind)- try:- prediction = program(**example.inputs())- except Exception as e:- print(f"Error processing example {example_ind}: {e}")- return None- # print("Getting trace...", example_ind)+ prediction = program(**example.inputs())trace = dspy.settings.trace- # print("Getting score...", example_ind)score = metric(example, prediction, trace) if metric else Nonedata_dict = {@@ -173,65 +159,3 @@ def process_example(example: Any, example_ind: int, program: Callable, metric: Odata_dict['score'] = scorereturn data_dict--# TODO: If we can ensure to pass the "round" information every time a call is-# issued to an LM, we can make repetitive un-cached calls to the same LM without-# modifying it's temperature. This function can be removed then.-def bootstrap_data_for_round(- program: Program,- dataset: List[Example],- metric: Optional[Callable[- [Example, Prediction, Optional[List]], Union[bool, int, float]- ]] = None,- num_threads = 1,- sampling_round: int = 0,- sampling_temperature: Optional[float] = 0.9,- sampling_temperature_delta: float = 0.001,- max_errors: int = 0- ) -> List[Dict]:- """ Bootstrap data for the given sampling round.-- This is a wrapper function around the `bootstrap_data` function that allows- for collecting data for the given `sampling_round`. Due to the way caching- works, one cannot get different completions for the same prompt just by- querying an LM again.- """- # TODO: [DSPy 2.5] Migrate to using "seed" for sampling- # Helper function to adjust the temperature of the LM. If a None temperature- # is passed, keep the LM's temperature as is as the base temperature, then- # adjust the temperature for the given round.- def copy_model_with_updated_temp(lm):- temp = sampling_temperature- temp = lm.kwargs["temperature"] if temp is None else temp- temp = temp + sampling_temperature_delta * sampling_round- return lm.copy(temperature=temp)-- # Ensure that the LM consistency is satisfied, which ensures that either (1)- # the global LM is set or (2) all the predictors have an LM set.- # TODO(isaac): Uncomment this line after the LM consistency property is- # program._assert_lm_consistency()-- # Deepcopy the program and copy the dataset to avoid modifying the original- program = program.deepcopy()- dataset = dataset.copy()-- # Update the temperature of the LM for the given round- context_lm = None- if dspy.settings.lm:- context_lm = copy_model_with_updated_temp(dspy.settings.lm)- else:- for pred in program.predictors():- pred.lm = copy_model_with_updated_temp(pred.lm)-- # Collect the data for the given round- with dspy.context(lm=context_lm):- # print(context_lm.kwargs)- data = bootstrap_data(- program, dataset, metric=metric, num_threads=num_threads, max_errors=max_errors- )-- # Add the round information to the data- for data_dict in data:- data_dict["round"] = sampling_round-- return data