Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add data-aware prompting to bayesian optimizer

Michael Ryan9fe0a5ac6f69
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.py
index dbe1b2468..dd976b07a 100644
--- a/dspy/teleprompt/signature_opt_bayesian.py
+++ b/dspy/teleprompt/signature_opt_bayesian.py
@@ -45,8 +45,34 @@ class BasicGenerateInstruction(Signature):
proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+class BasicGenerateInstructionWithDataObservations(Signature):
+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
+
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ observations = dspy.InputField(desc="Observations about the dataset and task")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class DatasetDescriptor(dspy.Signature):
+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
+
+ examples = dspy.InputField(desc="Sample data points from the dataset")
+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")
+
+class DatasetDescriptorWithPriorObservations(dspy.Signature):
+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """
+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
+
+ examples = dspy.InputField(desc="Sample data points from the dataset")
+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")
+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")
+
class BayesianSignatureOptimizer(Teleprompter):
- def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False):
+ def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False, view_data_batch_size=10):
self.n = n
self.metric = metric
self.init_temperature = init_temperature
@@ -55,6 +81,7 @@ class BayesianSignatureOptimizer(Teleprompter):
self.verbose = verbose
self.track_stats = track_stats
self.teacher_settings = teacher_settings
+ self.view_data_batch_size = view_data_batch_size
def _print_full_program(self, program):
for i,predictor in enumerate(program.predictors()):
@@ -70,11 +97,38 @@ class BayesianSignatureOptimizer(Teleprompter):
def _print_model_history(self, model, n=1):
if self.verbose: print(f"Model ({model}) History:")
model.inspect_history(n=n)
+
+ def _observe_data(self, trainset, data_limit=500):
+ upper_lim = min(len(trainset), self.view_data_batch_size)
+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))
+ observations = observation["observations"]
+
+ skips = 0
+ for b in range(self.view_data_batch_size, min(len(trainset), data_limit), self.view_data_batch_size):
+ upper_lim = min(min(len(trainset), data_limit), b+self.view_data_batch_size)
+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=(trainset[b:upper_lim].__repr__()))
+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":
+ skips += 1
+ if skips >= 5:
+ break
+ continue
+ observations += output["observations"]
+
+ return observations
- def _generate_first_N_candidates(self, module, N):
+ def _generate_first_N_candidates(self, module, N, data_informed_prompt_generation, devset):
candidates = {}
evaluated_candidates = defaultdict(dict)
+ if data_informed_prompt_generation:
+ # Create data observations
+ self.observations = None
+ if self.prompt_model:
+ with dspy.settings.context(lm=self.prompt_model):
+ self.observations = self._observe_data(devset).replace("Observations:","")
+ else:
+ self.observations = self._observe_data(devset).replace("Observations:","")
+
# Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
for predictor in module.predictors():
basic_instruction = None
@@ -86,10 +140,17 @@ class BayesianSignatureOptimizer(Teleprompter):
basic_instruction = predictor.extended_signature1.instructions
basic_prefix = predictor.extended_signature1.fields[-1].name
if self.prompt_model:
- with dspy.settings.context(lm=self.prompt_model):
- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+ if data_informed_prompt_generation:
+ with dspy.settings.context(lm=self.prompt_model):
+ instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)
+ else:
+ with dspy.settings.context(lm=self.prompt_model):
+ instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
else:
- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+ if data_informed_prompt_generation:
+ instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)
+ else:
+ instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
# Add in our initial prompt as a candidate as well
instruct.completions.proposed_instruction.append(basic_instruction)
instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)
@@ -100,7 +161,7 @@ class BayesianSignatureOptimizer(Teleprompter):
return candidates, evaluated_candidates
- def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42):
+ def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, data_informed_prompt_generation=True):
random.seed(seed)
@@ -109,7 +170,7 @@ class BayesianSignatureOptimizer(Teleprompter):
evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
# Generate N candidate prompts
- instruction_candidates, _ = self._generate_first_N_candidates(module, self.n)
+ instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, data_informed_prompt_generation, devset)
# Generate N few shot example sets
demo_candidates = {}