DSPy: The framework for programming—not prompting—language models
Python
Comments, type hints, etc. added to optimizers.
Robert P. Goldmanfc15eb2e32a0
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 8bbecef8f..682f524a3 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -1,5 +1,6 @@import randomimport threading+from typing import Optional, Dictimport tqdm@@ -15,7 +16,8 @@ from .vanilla import LabeledFewShot# TODO: Switch here from dsp.Example to dspy.Example. Right now, it's okay because it's internal only (predictors).# NOTE: Notice the places where we don't shuffle examples. I do like that this one doesn't shuffle.-# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when considering candidates.+# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when+# considering candidates.# TODO: the max_rounds via branch_idx to get past the cache, not just temperature.# In principle, we can also sample multiple outputs from the final generation step@@ -25,25 +27,54 @@ from .vanilla import LabeledFewShot# won't hurt our "best effort" guarantees.)# TODO: When this bootstraps for another teleprompter like finetune, we want all demos we gather.-# But when it's for direct use we may want to sample ONE demo per predictor--example pair. This is important for "multi-use" modules.+# But when it's for direct use we may want to sample ONE demo per predictor--example pair.+# This is important for "multi-use" modules.# TODO: Add baselines=[...]+# QUESTION: What is the meaning of self.validation and self.valset? Why is it that valset+# overrides validation if it is supplied? What is the relationship between the `valset`+# parameter to `compile` and the `trainset` parameter? I note that none of the examples in the+# docs seem to use this parameter.class BootstrapFewShot(Teleprompter):def __init__(self,metric=None,metric_threshold=None,- teacher_settings={},+ teacher_settings: Optional[Dict]=None,max_bootstrapped_demos=4,max_labeled_demos=16,max_rounds=1,max_errors=5,):+ """+ A Teleprompter class that composes a set of demos/examples to go into a predictor's prompt.+ These demos come from a combination of labeled examples in the training set, and newly-generated+ bootstrap demos.++ Parameters+ ----------+ metric: Callable+ A function that should be called on an `Example` and a `Prediction` and yield a value,+ which may be a `bool` or a number.+ metric_threshold: optional float, default `None`+ If the metric yields a numerical value, then check it against this threshold when+ deciding whether or not to accept a bootstrap example.+ teacher_settings: dict, optional+ Settings to go to the `teacher` model.+ max_bootstrapped_demos: int, default 4+ max_labeled_demos: int, default 16+ max_rounds: int, default 1+ This parameter controls the bootstrap-generation process. If one iteration through the+ training set is not enough to generate all the required bootstrap traces, then keep trying+ for `max_rounds` iterations, then give up.+ max_errors: int, default 5+ """self.metric = metricself.metric_threshold = metric_threshold- self.teacher_settings = teacher_settings+ ## linters didn't like having mutable arg as default value.+ self.teacher_settings = {} if teacher_settings is None else teacher_settingsself.max_bootstrapped_demos = max_bootstrapped_demosself.max_labeled_demos = max_labeled_demos@@ -69,13 +100,17 @@ class BootstrapFewShot(Teleprompter):return self.student+ ### Set self.student to a fresh copy of the student, and set teacher+ ### to a supplied value or to a copy of the student.def _prepare_student_and_teacher(self, student, teacher):self.student = student.reset_copy()self.teacher = teacher.deepcopy() if teacher is not None else student.reset_copy()- assert getattr(self.student, "_compiled", False) is False, "Student must be uncompiled."+ assert getattr(self.student, "_compiled", False) is False, "Student must not be compiled."if self.max_labeled_demos and getattr(self.teacher, "_compiled", False) is False:+ ## if we do not have a pretrained teacher, then we make a teacher by seeding the+ ## teacher (probably just a copy of the student) with a set of randomly-chosen demos.teleprompter = LabeledFewShot(k=self.max_labeled_demos)self.teacher = teleprompter.compile(self.teacher.reset_copy(), trainset=self.trainset)@@ -91,9 +126,14 @@ class BootstrapFewShot(Teleprompter):assert name1 == name2, "Student and teacher must have the same program structure."assert predictor1.signature.equals(predictor2.signature,- ), f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"+ ), (f"Student and teacher must have the same signatures. "+ f"{type(predictor1.signature)} != {type(predictor2.signature)}"+ )assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."+ ## FIXME: this seems very odd. `name2predictor` is a map from names to None.+ ## its only use seems to be getting iterated over, so I *think* it could be a+ ## set instead of a dict. - rpgname2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)predictor2name[id(predictor1)] = name1diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.pyindex 5b62d9283..1d1878ba9 100644--- a/dspy/teleprompt/copro_optimizer.py+++ b/dspy/teleprompt/copro_optimizer.py@@ -126,7 +126,23 @@ class COPRO(Teleprompter):predictor.signature = updated_signaturedef compile(self, student, *, trainset, eval_kwargs):- """student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""+ """+ Optimize the `signature` of `student`.+ `student` is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized+ (the demos are already chosen).++ Parameters+ ==========+ student:+ The program to optimize. Will not be modified.+ trainset: iterable of `Example`s+ eval_kwargs: optional, dict+ Additional keywords to go into `Evaluate` for the metric.++ Returns+ =======+ Optimized version of `student`.+ """module = student.deepcopy()evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)total_calls = 0diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.pyindex c9e34189f..e4dbe835e 100644--- a/dspy/teleprompt/mipro_optimizer.py+++ b/dspy/teleprompt/mipro_optimizer.py@@ -18,19 +18,21 @@ from dspy.teleprompt.teleprompt import Teleprompter"""USAGE SUGGESTIONS:-The following code can be used to compile a optimized signature teleprompter using the MIPRO, and evaluate it on an end task:+The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:+``` pythonfrom dspy.teleprompt import MIPROteleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)+```Note that this teleprompter takes in the following parameters:-* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).-* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).+* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).* metric: The task metric used for optimization.* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.pyindex ad2f6fe4a..eb0f6f74d 100644--- a/dspy/teleprompt/teleprompt_optuna.py+++ b/dspy/teleprompt/teleprompt_optuna.py@@ -52,6 +52,8 @@ class BootstrapFewShotWithOptuna(Teleprompter):display_table=False,display_progress=True,)+ # FIXME: this should have return_all_scores=False and then need only score,+ # since only one value will be returned. - rpgscore, _ = evaluate(program2, return_all_scores=True)trial.set_user_attr("program", program2)return score