DSPy: The framework for programming—not prompting—language models
Python
Fixed bug with repeat prompts, added stats
Michael Ryanabc35a82ef3c
diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.pyindex 80cad1517..f96e60fab 100644--- a/dspy/teleprompt/signature_opt.py+++ b/dspy/teleprompt/signature_opt.py@@ -3,6 +3,7 @@ import dspyfrom dspy.teleprompt.teleprompt import Teleprompterfrom dspy.signatures import Signaturefrom dspy.evaluate.evaluate import Evaluate+from collections import defaultdict"""USAGE SUGGESTIONS:@@ -22,7 +23,12 @@ Note that this teleprompter takes in the following parameters:* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.4.* prompt_model: The model used for prompt generation.* verbose: Tells the method whether or not to print intermediate steps.-+* track_stats: Tells the method whether or not to track statistics about the optimization process.+ If True, the method will track the following statistics:+ * results_best: The min,max,avg,stddev of top 10 scores for each predictor at each depth.+ * results_latest: The min,max,avg,stddev of newest prompt scores for each predictor at each depth.+ * total_calls: The total number of calls to the task metric.+ These statistics will be returned as attributes of the best program."""class BasicGenerateInstruction(Signature):"""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""@@ -31,21 +37,24 @@ class BasicGenerateInstruction(Signature):proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")-class GenerateInstructionGivenAttempts(Signature):- """You are an experienced instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality. Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative."""+class GenerateInstructionGivenAttempts(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality.++Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative."""attempted_instructions = dspy.InputField(format=dsp.passages2text)proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")class SignatureOptimizer(Teleprompter):- def __init__(self, metric=None, breadth=10, depth=3, init_temperature=1.4, prompt_model="gpt-3.5-turbo-1106", verbose=False):+ def __init__(self, metric=None, breadth=10, depth=3, init_temperature=1.4, prompt_model="gpt-3.5-turbo-1106", verbose=False, track_stats=False):self.metric = metricself.breadth = breadthself.depth = depthself.init_temperature = init_temperatureself.prompt_model = prompt_modelself.verbose = verbose+ self.track_stats = track_statsdef _check_candidates_equal(self, candidate1, candidate2):for p1, p2 in zip(candidate1["program"].predictors(), candidate2["program"].predictors()):@@ -77,25 +86,38 @@ class SignatureOptimizer(Teleprompter):def compile(self, student, *, devset, eval_kwargs):"""student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""-module = student.deepcopy()evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)total_calls = 0+ results_best = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}+ results_latest = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}++ if self.track_stats:+ import numpy as np+candidates = {}- evaluated_candidates = {}+ evaluated_candidates = defaultdict(dict)# Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new promptsfor predictor in module.predictors():- basic_instruction = predictor.extended_signature.instructions- basic_prefix = predictor.extended_signature.fields[-1].name+ basic_instruction = None+ basic_prefix = None+ if (hasattr(predictor, 'extended_signature')):+ basic_instruction = predictor.extended_signature.instructions+ basic_prefix = predictor.extended_signature.fields[-1].name+ else:+ basic_instruction = predictor.extended_signature1.instructions+ basic_prefix = predictor.extended_signature1.fields[-1].namewith dspy.settings.context(lm=self.prompt_model):instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)# Add in our initial prompt as a candidate as wellinstruct.completions.proposed_instruction.append(basic_instruction)instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)candidates[id(predictor)] = instruct.completions- evaluated_candidates[id(predictor)] = []+ evaluated_candidates[id(predictor)] = {}++ if self.verbose: print(f"{self.prompt_model.inspect_history(n=1)}")latest_candidates = candidatesall_candidates = candidates@@ -103,108 +125,167 @@ class SignatureOptimizer(Teleprompter):module_clone = module.deepcopy()# For each iteration in depth...- for d in range(self.depth):- if (self.verbose):- print(f"Starting iteration {d}/{self.depth}.")+ for d in range(self.depth): # TODO: fix this so that we eval the new batch of predictors with the new best followoing predictors+ if self.verbose: print(f"Starting iteration {d}/{self.depth}.")++ latest_scores = []# Go through our module's predictorsfor p_old, p_new in zip(module.predictors(), module_clone.predictors()):+candidates_ = latest_candidates[id(p_old)] # Use the most recently generated candidates for evaluationif len(module.predictors()) > 1:candidates_ = all_candidates[id(p_old)] # Unless our program has multiple predictors, in which case we need to reevaluate all prompts with the new prompt(s) for the other predictor(s)# For each candidate- for c in candidates_:+ for c_num, c in enumerate(candidates_):# Get the candidate instruction and prefixinstruction, prefix = c.proposed_instruction.strip('"').strip(), c.proposed_prefix_for_output_field.strip('"').strip()# Set this new module with our instruction / prefix- p_new.extended_signature.instructions = instruction- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=prefix)+ if (hasattr(p_new, 'extended_signature')):+ p_new.extended_signature.instructions = instruction+ p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=prefix)+ else:+ p_new.extended_signature1.instructions = instruction+ p_new.extended_signature1.fields[-1] = p_new.extended_signature1.fields[-1]._replace(name=prefix)+ p_new.extended_signature2.instructions = instruction+ p_new.extended_signature2.fields[-1] = p_new.extended_signature2.fields[-1]._replace(name=prefix)# Score the instruction / prefix- if (self.verbose):- print(f"----------------")- for i,predictor in enumerate(module_clone.predictors()):- print(f"Predictor {i}")- print(f"i: {predictor.extended_signature.instructions}")- print(f"p: {predictor.extended_signature.fields[-1].name}")- print()+ if self.verbose: print(f"----------------")+ for i,predictor in enumerate(module_clone.predictors()):+ if self.verbose: print(f"Predictor {i}")+ if (hasattr(predictor, 'extended_signature')):+ if self.verbose: print(f"i: {predictor.extended_signature.instructions}")+ if self.verbose: print(f"p: {predictor.extended_signature.fields[-1].name}")+ else:+ if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")+ if self.verbose: print(f"p: {predictor.extended_signature1.fields[-1].name}")+ if self.verbose: print()score = evaluate(module_clone, devset=devset, **eval_kwargs)+ print(f"prompt_model.inspect_history(n=1) {self.prompt_model.inspect_history(n=1)}")total_calls += 1- if (self.verbose):- print(f"----------------")-- # Add it to our evaluated candidates list- evaluated_candidates[id(p_old)].append({- "score": score,- "program": module_clone.deepcopy(),- "instruction": instruction,- "prefix": prefix,- "depth": d- })+ if self.verbose: print(f"----------------")++ replace_entry = True+ if self.verbose: print(f"(instruction, prefix) {(instruction, prefix)}")+ # if verbose: print(f"evaluated_candidates[id(p_old)] {evaluated_candidates[id(p_old)]}")+ if ((instruction, prefix) in evaluated_candidates[id(p_old)]):+ # if verbose: print(f"if evaluated_candidates[id(p_old)][(instruction, prefix)] {evaluated_candidates[id(p_old)][(instruction, prefix)]}")+ if evaluated_candidates[id(p_old)][(instruction, prefix)]["score"] >= score:+ replace_entry = False++ if replace_entry:+ # Add it to our evaluated candidates list+ evaluated_candidates[id(p_old)][(instruction, prefix)] = {+ "score": score,+ "program": module_clone.deepcopy(),+ "instruction": instruction,+ "prefix": prefix,+ "depth": d+ }++ if (len(candidates_)-self.breadth <= c_num):+ latest_scores.append(score)++ if self.track_stats:+ results_latest[id(p_old)]["depth"].append(d)+ results_latest[id(p_old)]["max"].append(max(latest_scores))+ results_latest[id(p_old)]["average"].append(sum(latest_scores)/len(latest_scores))+ results_latest[id(p_old)]["min"].append(min(latest_scores))+ results_latest[id(p_old)]["std"].append(np.std(latest_scores))# Now that we've evaluated the candidates, set this predictor to the best performing version# to ensure the next round of scores reflect the best possible version- best_candidate = max(evaluated_candidates[id(p_old)], key=lambda candidate: candidate['score'])- p_new.extended_signature.instructions = best_candidate["instruction"]- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=best_candidate["prefix"])- if (self.verbose):- print(f"Updating Predictor {id(p_old)} to:\ni: {best_candidate['instruction']}\np: {best_candidate['prefix']}")- print(f"Full predictor with update: ")- for i,predictor in enumerate(module_clone.predictors()):- print(f"Predictor {i}")- print(f"i: {predictor.extended_signature.instructions}")- print(f"p: {predictor.extended_signature.fields[-1].name}")- print()-- for predictor_candidate_list in evaluated_candidates.values():- predictor_candidate_list.sort(key=lambda x: x['score'], reverse=True)+ best_candidate = max(evaluated_candidates[id(p_old)].values(), key=lambda candidate: candidate['score'])+ if (hasattr(p_new, 'extended_signature')):+ p_new.extended_signature.instructions = best_candidate["instruction"]+ p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=best_candidate["prefix"])+ else:+ p_new.extended_signature1.instructions = best_candidate["instruction"]+ p_new.extended_signature1.fields[-1] = p_new.extended_signature1.fields[-1]._replace(name=best_candidate["prefix"])+ p_new.extended_signature2.instructions = best_candidate["instruction"]+ p_new.extended_signature2.fields[-1] = p_new.extended_signature2.fields[-1]._replace(name=best_candidate["prefix"])+ if self.verbose: print(f"Updating Predictor {id(p_old)} to:\ni: {best_candidate['instruction']}\np: {best_candidate['prefix']}")+ if self.verbose: print(f"Full predictor with update: ")+ for i,predictor in enumerate(module_clone.predictors()):+ if self.verbose: print(f"Predictor {i}")+ if (hasattr(predictor, 'extended_signature')):+ if self.verbose: print(f"i: {predictor.extended_signature.instructions}")+ if self.verbose: print(f"p: {predictor.extended_signature.fields[-1].name}")+ else:+ if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")+ if self.verbose: print(f"p: {predictor.extended_signature1.fields[-1].name}")+ if self.verbose: print()if d == self.depth-1:break- # Build Few-Shot Example of Optimized Prompts- attempts = {}- shortest_len = self.breadth- for p in module.predictors():- attempts[id(p)] = [] # Initialize as empty list- shortest_len = min(len(evaluated_candidates[id(p)]),shortest_len)-- for i in range(shortest_len-1,-1,-1):- for p_base in module.predictors():- attempts[id(p_base)].append(f'Instruction #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["instruction"]}')- attempts[id(p_base)].append(f'Prefix #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["prefix"]}')- attempts[id(p_base)].append(f'Resulting Score #{shortest_len-i}: {evaluated_candidates[id(p_base)][i]["score"]}')- # Generate next batch of potential prompts to optimize, with previous attempts as inputnew_candidates = {}- for predictor in module.predictors():+ for p_base in module.predictors():+ # Build Few-Shot Example of Optimized Prompts+ attempts = []+ shortest_len = self.breadth+ shortest_len = min(len(evaluated_candidates[id(p_base)]),shortest_len)+ best_predictors = list(evaluated_candidates[id(p_base)].values())++ # best_predictors = evaluated_candidates[id(p_base)].values()[:]+ best_predictors.sort(key=lambda x: x['score'], reverse=True)++ if self.track_stats:+ scores = [x['score'] for x in best_predictors][:10]+ results_best[id(p_base)]["depth"].append(d)+ results_best[id(p_base)]["max"].append(max(scores))+ results_best[id(p_base)]["average"].append(sum(scores)/len(scores))+ results_best[id(p_base)]["min"].append(min(scores))+ results_best[id(p_base)]["std"].append(np.std(scores))++ for i in range(shortest_len-1,-1,-1):+ # breakpoint()+ attempts.append(f'Instruction #{shortest_len-i}: {best_predictors[i]["instruction"]}')+ attempts.append(f'Prefix #{shortest_len-i}: {best_predictors[i]["prefix"]}')+ attempts.append(f'Resulting Score #{shortest_len-i}: {best_predictors[i]["score"]}')++ # Generate next batch of potential prompts to optimize, with previous attempts as inputwith dspy.settings.context(lm=self.prompt_model):- instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts[id(predictor)])- if (self.verbose):- print(f"{self.prompt_model.inspect_history(n=1)}")+ instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts)+ if self.verbose: print(f"{self.prompt_model.inspect_history(n=1)}")# Get candidates for each predictor- new_candidates[id(predictor)] = instr.completions- all_candidates[id(predictor)].proposed_instruction.extend(instr.completions.proposed_instruction)- all_candidates[id(predictor)].proposed_prefix_for_output_field.extend(instr.completions.proposed_prefix_for_output_field)+ new_candidates[id(p_base)] = instr.completions+ all_candidates[id(p_base)].proposed_instruction.extend(instr.completions.proposed_instruction)+ all_candidates[id(p_base)].proposed_prefix_for_output_field.extend(instr.completions.proposed_prefix_for_output_field)+ if self.verbose: print(f"{self.prompt_model.inspect_history(n=1)}")latest_candidates = new_candidates-- for candidate_list in evaluated_candidates.values():- candidate_list.sort(key=lambda x: x['score'], reverse=True)-- best_program = evaluated_candidates[id(p_old)][0]["program"] # Uses python loose scoping to get the best candidate from the last round-+candidates = []for predictor in module.predictors():- candidates.extend(evaluated_candidates[id(predictor)])+ candidates.extend(list(evaluated_candidates[id(predictor)].values()))+ if self.track_stats:+ best_predictors = list(evaluated_candidates[id(predictor)].values())+ best_predictors.sort(key=lambda x: x['score'], reverse=True)++ scores = [x['score'] for x in best_predictors][:10]+ results_best[id(predictor)]["depth"].append(d)+ results_best[id(predictor)]["max"].append(max(scores))+ results_best[id(predictor)]["average"].append(sum(scores)/len(scores))+ results_best[id(predictor)]["min"].append(min(scores))+ results_best[id(predictor)]["std"].append(np.std(scores))++ # if verbose: print(f"candidates: {candidates}")candidates.sort(key=lambda x: x['score'], reverse=True)+candidates = self._drop_duplicates(candidates)+ best_program = candidates[0]["program"]best_program.candidate_programs = candidatesbest_program.total_calls = total_calls+ if self.track_stats:+ best_program.results_best = results_best+ best_program.results_latest = results_latestreturn best_program\ No newline at end of file