DSPy: The framework for programming—not prompting—language models
Python
using 3 examples in instruction gen prompt to avoid overfitting to one example (#1964)
klopsahlongc2bc3188f130
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.pyindex 15ef13130..94ac1783f 100644--- a/dspy/propose/grounded_proposer.py+++ b/dspy/propose/grounded_proposer.py@@ -5,7 +5,7 @@ from dspy.propose.dataset_summary_generator import create_dataset_summaryfrom dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix, get_dspy_source_codefrom dspy.teleprompt.utils import get_signature, get_prompt_model-from dspy.propose.propose_base import Proposer+from .propose_base import Proposer# Hardcoded variables (TODO: update)MAX_INSTRUCT_IN_HISTORY = 5 # 10@@ -97,6 +97,9 @@ def generate_instruction_class(module = dspy.InputField(desc="The module to create an instruction for.", prefix="MODULE:",)+ module_description = dspy.InputField(+ desc="Description of the module to create an instruction for.", prefix="MODULE DESCRIPTION:",+ )task_demos = dspy.InputField(format=str,desc="Example inputs/outputs of our module.",@@ -164,24 +167,39 @@ class GenerateModuleInstruction(dspy.Module):program,previous_instructions,data_summary,- max_demos=3,+ num_demos_in_context=3,tip=None,):- # Construct full program demo or single module demo depending on whether or not we're using the full program- task_demos = ""+ def gather_examples_from_sets(candidate_sets, max_examples):+ """Helper function to gather up to augmented examples from given sets."""+ count = 0+ for candidate_set in candidate_sets:+ for example in candidate_set:+ if "augmented" in example.keys():+ fields_to_use = get_signature(program.predictors()[pred_i]).fields+ yield create_example_string(fields_to_use, example)+ count += 1+ if count >= max_examples:+ return++ # Construct full program demo or single module demo depending on settingsbasic_instruction = get_signature(program.predictors()[pred_i]).instructions- curr_demos_num = 0+ task_demos = ""if self.use_task_demos:- for example in demo_candidates[pred_i][demo_set_i]:- if "augmented" in example.keys():- fields_to_use = get_signature(program.predictors()[pred_i]).fields- example_string = create_example_string(fields_to_use, example)- task_demos += f"{example_string}\n"- curr_demos_num += 1- if curr_demos_num >= max_demos:- break- else:+ # Combine current and adjacent sets+ adjacent_sets = (+ [demo_candidates[pred_i][demo_set_i]] ++ demo_candidates[pred_i][demo_set_i + 1:] ++ demo_candidates[pred_i][:demo_set_i]+ )++ # Gather examples up to the required count+ example_strings = gather_examples_from_sets(adjacent_sets, num_demos_in_context)+ task_demos = "\n\n".join(example_strings) + "\n\n"++ # Default to no demos provided if no examples were gathered, or if we're using the first demo set+ if not task_demos.strip() or demo_set_i == 0:task_demos = "No task demos provided."# Summarize the program@@ -224,20 +242,19 @@ class GenerateModuleInstruction(dspy.Module):# Generate an instruction for our chosen moduleif self.verbose: print(f"task_demos {task_demos}")+instruct = self.generate_module_instruction(dataset_description=data_summary,program_code=self.program_code_string,- program_description=program_description,module=module_code,+ program_description=program_description,+ module_description=module_description,task_demos=task_demos,tip=tip,basic_instruction=basic_instruction,previous_instructions=previous_instructions,- module_description=module_description,)- if hasattr(instruct, "module_description"):- module_description = strip_prefix(instruct.module_description)- if self.verbose: print(f"MODULE DESCRIPTION: {module_description}")+proposed_instruction = strip_prefix(instruct.proposed_instruction)return dspy.Prediction(proposed_instruction=proposed_instruction)@@ -254,6 +271,7 @@ class GroundedProposer(Proposer):use_dataset_summary=True,program_aware=True,use_task_demos=True,+ num_demos_in_context = 3,use_instruct_history=True,use_tip=True,set_tip_randomly=True,@@ -265,6 +283,7 @@ class GroundedProposer(Proposer):self.program_aware = program_awareself.use_dataset_summary = use_dataset_summaryself.use_task_demos = use_task_demos+ self.num_demos_in_context = num_demos_in_contextself.use_instruct_history = use_instruct_historyself.use_tip = use_tipself.set_tip_randomly=set_tip_randomly@@ -315,15 +334,13 @@ class GroundedProposer(Proposer):self.use_instruct_history = use_historyif self.verbose: print(f"Use history T/F: {self.use_instruct_history}")- num_demos = max(len(demo_candidates[0]) if demo_candidates else N, 1)-if not demo_candidates:if self.verbose: print("No demo candidates provided. Running without task demos.")self.use_task_demos = False# Create an instruction for each predictorfor pred_i, predictor in enumerate(program.predictors()):- for demo_set_i in range(num_demos):+ for demo_set_i in range(len(demo_candidates[0])):if pred_i not in proposed_instructions:proposed_instructions[pred_i] = []if self.set_tip_randomly:@@ -341,7 +358,6 @@ class GroundedProposer(Proposer):program=program,predictor=predictor,pred_i=pred_i,- # prompt_model=prompt_model,T=T,demo_candidates=demo_candidates,demo_set_i=demo_set_i,@@ -349,7 +365,7 @@ class GroundedProposer(Proposer):tip=selected_tip,),)-+return proposed_instructionsdef propose_instruction_for_predictor(@@ -385,8 +401,11 @@ class GroundedProposer(Proposer):# Generate a new instruction for our predictor, using the temperature specified for this roundoriginal_temp = self.prompt_model.kwargs["temperature"]+ epsilon = self.rng.uniform(0.01, 0.05)+ modified_temp = T + epsilon+with dspy.settings.context(lm=self.prompt_model):- self.prompt_model.kwargs["temperature"] = T+ self.prompt_model.kwargs["temperature"] = modified_tempproposed_instruction = instruction_generator.forward(demo_candidates=demo_candidates,pred_i=pred_i,@@ -394,6 +413,7 @@ class GroundedProposer(Proposer):program=program,data_summary=self.data_summary,previous_instructions=instruction_history,+ num_demos_in_context = self.num_demos_in_context,tip=tip,).proposed_instructionself.prompt_model.kwargs["temperature"] = original_temp@@ -402,4 +422,4 @@ class GroundedProposer(Proposer):if self.verbose: self.prompt_model.inspect_history(n=1)if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")- return strip_prefix(proposed_instruction)+ return strip_prefix(proposed_instruction)\ No newline at end of filediff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex 5ac7f1dff..69314331d 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -437,6 +437,7 @@ class MIPROv2(Teleprompter):program_aware=program_aware_proposer,use_dataset_summary=data_aware_proposer,use_task_demos=fewshot_aware_proposer,+ num_demos_in_context = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT,use_tip=tip_aware_proposer,set_tip_randomly=tip_aware_proposer,use_instruct_history=False,