Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

finishing touches to signature_opt.py + usage instructions

Krista Opsahl-Ong01f532cb0686
diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.py
index 9244fbbd3..90ee99122 100644
--- a/dspy/teleprompt/signature_opt.py
+++ b/dspy/teleprompt/signature_opt.py
@@ -1,13 +1,28 @@
import dsp
import dspy
-import pickle
-
from dspy.teleprompt.teleprompt import Teleprompter
from dspy.signatures import Signature
-
from dspy.evaluate.evaluate import Evaluate
-# Define teleprompter
+"""
+USAGE SUGGESTIONS:
+
+The following code can be used to compile a optimized signature teleprompter, and evaluate it on an end task:
+
+teleprompter = SignatureOptimizer(metric=metric, breadth=BREADTH, depth=DEPTH, init_temperature=INIT_TEMPERATURE, prompt_model=prompt_model, output_dir=optimizedV1_output_dir)
+compiled_prompt_opt = teleprompter.compile(program.deepcopy(), devset=devset[:DEV_NUM], eval_kwargs=kwargs)
+eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
+
+Note that this teleprompter takes in the following parameters:
+
+* metric: The task metric used for optimization.
+* breadth: The number of new prompts to generate at each iteration. Default=10.
+* depth: The number of times we should ask our prompt model to genereate new prompts, with the history of the past prompts as input. Default=3.
+* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.4.
+* prompt_model: The model used for prompt generation.
+* verbose: Whether or not to print intermediate steps.
+
+"""
class BasicGenerateInstruction(Signature):
"""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
@@ -23,13 +38,12 @@ class GenerateInstructionGivenAttempts(Signature):
proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
class SignatureOptimizer(Teleprompter):
- def __init__(self, metric=None, breadth=10, depth=3, init_temperature=1.4, prompt_model="gpt-3.5-turbo-1106", output_file=None, verbose=False):
+ def __init__(self, metric=None, breadth=10, depth=3, init_temperature=1.4, prompt_model="gpt-3.5-turbo-1106", verbose=False):
self.metric = metric
self.breadth = breadth
self.depth = depth
self.init_temperature = init_temperature
self.prompt_model = prompt_model
- self.output_file = output_file
self.verbose = verbose
def _check_candidates_equal(self, candidate1, candidate2):
@@ -62,6 +76,7 @@ class SignatureOptimizer(Teleprompter):
def compile(self, student, *, devset, eval_kwargs):
"""student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""
+
module = student.deepcopy()
evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
total_calls = 0
@@ -75,28 +90,27 @@ class SignatureOptimizer(Teleprompter):
basic_prefix = predictor.extended_signature.fields[-1].name
with dspy.settings.context(lm=self.prompt_model):
instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+ # Add in our initial prompt as a candidate as well
instruct.completions.proposed_instruction.append(basic_instruction)
instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)
candidates[id(predictor)] = instruct.completions
evaluated_candidates[id(predictor)] = []
- last_best_programs = []
- seen = set()
latest_candidates = candidates
all_candidates = candidates
module_clone = module.deepcopy()
# For each iteration in depth...
- for d in range(self.depth): # TODO: fix this so that we eval the new batch of predictors with the new best followoing predictors
+ for d in range(self.depth):
if (self.verbose):
print(f"Starting iteration {d}/{self.depth}.")
# Go through our module's predictors
for p_old, p_new in zip(module.predictors(), module_clone.predictors()):
-
- # Get the latest round of prompt candidates for current predictor
- candidates_ = all_candidates[id(p_old)]
+ candidates_ = latest_candidates[id(p_old)] # Use the most recently generated candidates for evaluation
+ if len(module.predictors()) > 1:
+ candidates_ = all_candidates[id(p_old)] # Unless our program has multiple predictors, in which case we need to reevaluate all prompts with the new prompt(s) for the other predictor(s)
# For each candidate
for c in candidates_:
@@ -159,7 +173,6 @@ class SignatureOptimizer(Teleprompter):
for i in range(shortest_len-1,-1,-1):
for p_base in module.predictors():
- # breakpoint()
attempts[id(p_base)].append(f'Instruction #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["instruction"]}')
attempts[id(p_base)].append(f'Prefix #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["prefix"]}')
attempts[id(p_base)].append(f'Resulting Score #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["score"]}')
@@ -180,11 +193,6 @@ class SignatureOptimizer(Teleprompter):
for candidate_list in evaluated_candidates.values():
candidate_list.sort(key=lambda x: x['score'], reverse=True)
-
- # Save the evaluated candidates using pickle
- if (self.output_file is not None):
- with open(self.output_dir, 'wb') as file:
- pickle.dump(evaluated_candidates, file)
best_program = evaluated_candidates[id(p_old)][0]["program"] # Uses python loose scoping to get the best candidate from the last round