Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Comments, type hints, etc. added to optimizers.

Robert P. Goldmanfc15eb2e32a0
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 8bbecef8f..682f524a3 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -1,5 +1,6 @@
import random
import threading
+from typing import Optional, Dict
import tqdm
@@ -15,7 +16,8 @@ from .vanilla import LabeledFewShot
# TODO: Switch here from dsp.Example to dspy.Example. Right now, it's okay because it's internal only (predictors).
# NOTE: Notice the places where we don't shuffle examples. I do like that this one doesn't shuffle.
-# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when considering candidates.
+# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when
+# considering candidates.
# TODO: the max_rounds via branch_idx to get past the cache, not just temperature.
# In principle, we can also sample multiple outputs from the final generation step
@@ -25,25 +27,54 @@ from .vanilla import LabeledFewShot
# won't hurt our "best effort" guarantees.)
# TODO: When this bootstraps for another teleprompter like finetune, we want all demos we gather.
-# But when it's for direct use we may want to sample ONE demo per predictor--example pair. This is important for "multi-use" modules.
+# But when it's for direct use we may want to sample ONE demo per predictor--example pair.
+# This is important for "multi-use" modules.
# TODO: Add baselines=[...]
+# QUESTION: What is the meaning of self.validation and self.valset? Why is it that valset
+# overrides validation if it is supplied? What is the relationship between the `valset`
+# parameter to `compile` and the `trainset` parameter? I note that none of the examples in the
+# docs seem to use this parameter.
class BootstrapFewShot(Teleprompter):
def __init__(
self,
metric=None,
metric_threshold=None,
- teacher_settings={},
+ teacher_settings: Optional[Dict]=None,
max_bootstrapped_demos=4,
max_labeled_demos=16,
max_rounds=1,
max_errors=5,
):
+ """
+ A Teleprompter class that composes a set of demos/examples to go into a predictor's prompt.
+ These demos come from a combination of labeled examples in the training set, and newly-generated
+ bootstrap demos.
+
+ Parameters
+ ----------
+ metric: Callable
+ A function that should be called on an `Example` and a `Prediction` and yield a value,
+ which may be a `bool` or a number.
+ metric_threshold: optional float, default `None`
+ If the metric yields a numerical value, then check it against this threshold when
+ deciding whether or not to accept a bootstrap example.
+ teacher_settings: dict, optional
+ Settings to go to the `teacher` model.
+ max_bootstrapped_demos: int, default 4
+ max_labeled_demos: int, default 16
+ max_rounds: int, default 1
+ This parameter controls the bootstrap-generation process. If one iteration through the
+ training set is not enough to generate all the required bootstrap traces, then keep trying
+ for `max_rounds` iterations, then give up.
+ max_errors: int, default 5
+ """
self.metric = metric
self.metric_threshold = metric_threshold
- self.teacher_settings = teacher_settings
+ ## linters didn't like having mutable arg as default value.
+ self.teacher_settings = {} if teacher_settings is None else teacher_settings
self.max_bootstrapped_demos = max_bootstrapped_demos
self.max_labeled_demos = max_labeled_demos
@@ -69,13 +100,17 @@ class BootstrapFewShot(Teleprompter):
return self.student
+ ### Set self.student to a fresh copy of the student, and set teacher
+ ### to a supplied value or to a copy of the student.
def _prepare_student_and_teacher(self, student, teacher):
self.student = student.reset_copy()
self.teacher = teacher.deepcopy() if teacher is not None else student.reset_copy()
- assert getattr(self.student, "_compiled", False) is False, "Student must be uncompiled."
+ assert getattr(self.student, "_compiled", False) is False, "Student must not be compiled."
if self.max_labeled_demos and getattr(self.teacher, "_compiled", False) is False:
+ ## if we do not have a pretrained teacher, then we make a teacher by seeding the
+ ## teacher (probably just a copy of the student) with a set of randomly-chosen demos.
teleprompter = LabeledFewShot(k=self.max_labeled_demos)
self.teacher = teleprompter.compile(self.teacher.reset_copy(), trainset=self.trainset)
@@ -91,9 +126,14 @@ class BootstrapFewShot(Teleprompter):
assert name1 == name2, "Student and teacher must have the same program structure."
assert predictor1.signature.equals(
predictor2.signature,
- ), f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"
+ ), (f"Student and teacher must have the same signatures. "
+ f"{type(predictor1.signature)} != {type(predictor2.signature)}"
+ )
assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."
+ ## FIXME: this seems very odd. `name2predictor` is a map from names to None.
+ ## its only use seems to be getting iterated over, so I *think* it could be a
+ ## set instead of a dict. - rpg
name2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)
predictor2name[id(predictor1)] = name1
diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.py
index 5b62d9283..1d1878ba9 100644
--- a/dspy/teleprompt/copro_optimizer.py
+++ b/dspy/teleprompt/copro_optimizer.py
@@ -126,7 +126,23 @@ class COPRO(Teleprompter):
predictor.signature = updated_signature
def compile(self, student, *, trainset, eval_kwargs):
- """student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""
+ """
+ Optimize the `signature` of `student`.
+ `student` is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized
+ (the demos are already chosen).
+
+ Parameters
+ ==========
+ student:
+ The program to optimize. Will not be modified.
+ trainset: iterable of `Example`s
+ eval_kwargs: optional, dict
+ Additional keywords to go into `Evaluate` for the metric.
+
+ Returns
+ =======
+ Optimized version of `student`.
+ """
module = student.deepcopy()
evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)
total_calls = 0
diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.py
index c9e34189f..e4dbe835e 100644
--- a/dspy/teleprompt/mipro_optimizer.py
+++ b/dspy/teleprompt/mipro_optimizer.py
@@ -18,19 +18,21 @@ from dspy.teleprompt.teleprompt import Teleprompter
"""
USAGE SUGGESTIONS:
-The following code can be used to compile a optimized signature teleprompter using the MIPRO, and evaluate it on an end task:
+The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:
+``` python
from dspy.teleprompt import MIPRO
teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)
kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
+```
Note that this teleprompter takes in the following parameters:
-* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
-* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
+* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).
+* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).
* metric: The task metric used for optimization.
* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.
* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.
diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.py
index ad2f6fe4a..eb0f6f74d 100644
--- a/dspy/teleprompt/teleprompt_optuna.py
+++ b/dspy/teleprompt/teleprompt_optuna.py
@@ -52,6 +52,8 @@ class BootstrapFewShotWithOptuna(Teleprompter):
display_table=False,
display_progress=True,
)
+ # FIXME: this should have return_all_scores=False and then need only score,
+ # since only one value will be returned. - rpg
score, _ = evaluate(program2, return_all_scores=True)
trial.set_user_attr("program", program2)
return score