DSPy: The framework for programming—not prompting—language models
Python
Add data-aware prompting to bayesian optimizer
Michael Ryan9fe0a5ac6f69
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.pyindex dbe1b2468..dd976b07a 100644--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@ -45,8 +45,34 @@ class BasicGenerateInstruction(Signature):proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")+class BasicGenerateInstructionWithDataObservations(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ observations = dspy.InputField(desc="Observations about the dataset and task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class DatasetDescriptor(dspy.Signature):+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")++ examples = dspy.InputField(desc="Sample data points from the dataset")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")++class DatasetDescriptorWithPriorObservations(dspy.Signature):+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")++ examples = dspy.InputField(desc="Sample data points from the dataset")+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")+class BayesianSignatureOptimizer(Teleprompter):- def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False):+ def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False, view_data_batch_size=10):self.n = nself.metric = metricself.init_temperature = init_temperature@@ -55,6 +81,7 @@ class BayesianSignatureOptimizer(Teleprompter):self.verbose = verboseself.track_stats = track_statsself.teacher_settings = teacher_settings+ self.view_data_batch_size = view_data_batch_sizedef _print_full_program(self, program):for i,predictor in enumerate(program.predictors()):@@ -70,11 +97,38 @@ class BayesianSignatureOptimizer(Teleprompter):def _print_model_history(self, model, n=1):if self.verbose: print(f"Model ({model}) History:")model.inspect_history(n=n)++ def _observe_data(self, trainset, data_limit=500):+ upper_lim = min(len(trainset), self.view_data_batch_size)+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))+ observations = observation["observations"]++ skips = 0+ for b in range(self.view_data_batch_size, min(len(trainset), data_limit), self.view_data_batch_size):+ upper_lim = min(min(len(trainset), data_limit), b+self.view_data_batch_size)+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=(trainset[b:upper_lim].__repr__()))+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":+ skips += 1+ if skips >= 5:+ break+ continue+ observations += output["observations"]++ return observations- def _generate_first_N_candidates(self, module, N):+ def _generate_first_N_candidates(self, module, N, data_informed_prompt_generation, devset):candidates = {}evaluated_candidates = defaultdict(dict)+ if data_informed_prompt_generation:+ # Create data observations+ self.observations = None+ if self.prompt_model:+ with dspy.settings.context(lm=self.prompt_model):+ self.observations = self._observe_data(devset).replace("Observations:","")+ else:+ self.observations = self._observe_data(devset).replace("Observations:","")+# Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new promptsfor predictor in module.predictors():basic_instruction = None@@ -86,10 +140,17 @@ class BayesianSignatureOptimizer(Teleprompter):basic_instruction = predictor.extended_signature1.instructionsbasic_prefix = predictor.extended_signature1.fields[-1].nameif self.prompt_model:- with dspy.settings.context(lm=self.prompt_model):- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)+ if data_informed_prompt_generation:+ with dspy.settings.context(lm=self.prompt_model):+ instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)+ else:+ with dspy.settings.context(lm=self.prompt_model):+ instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)else:- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)+ if data_informed_prompt_generation:+ instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)+ else:+ instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)# Add in our initial prompt as a candidate as wellinstruct.completions.proposed_instruction.append(basic_instruction)instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)@@ -100,7 +161,7 @@ class BayesianSignatureOptimizer(Teleprompter):return candidates, evaluated_candidates- def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42):+ def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, data_informed_prompt_generation=True):random.seed(seed)@@ -109,7 +170,7 @@ class BayesianSignatureOptimizer(Teleprompter):evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)# Generate N candidate prompts- instruction_candidates, _ = self._generate_first_N_candidates(module, self.n)+ instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, data_informed_prompt_generation, devset)# Generate N few shot example setsdemo_candidates = {}