Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

using 3 examples in instruction gen prompt to avoid overfitting to one example (#1964)

klopsahlongc2bc3188f130
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.py
index 15ef13130..94ac1783f 100644
--- a/dspy/propose/grounded_proposer.py
+++ b/dspy/propose/grounded_proposer.py
@@ -5,7 +5,7 @@ from dspy.propose.dataset_summary_generator import create_dataset_summary
from dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix, get_dspy_source_code
from dspy.teleprompt.utils import get_signature, get_prompt_model
-from dspy.propose.propose_base import Proposer
+from .propose_base import Proposer
# Hardcoded variables (TODO: update)
MAX_INSTRUCT_IN_HISTORY = 5 # 10
@@ -97,6 +97,9 @@ def generate_instruction_class(
module = dspy.InputField(
desc="The module to create an instruction for.", prefix="MODULE:",
)
+ module_description = dspy.InputField(
+ desc="Description of the module to create an instruction for.", prefix="MODULE DESCRIPTION:",
+ )
task_demos = dspy.InputField(
format=str,
desc="Example inputs/outputs of our module.",
@@ -164,24 +167,39 @@ class GenerateModuleInstruction(dspy.Module):
program,
previous_instructions,
data_summary,
- max_demos=3,
+ num_demos_in_context=3,
tip=None,
):
- # Construct full program demo or single module demo depending on whether or not we're using the full program
- task_demos = ""
+ def gather_examples_from_sets(candidate_sets, max_examples):
+ """Helper function to gather up to augmented examples from given sets."""
+ count = 0
+ for candidate_set in candidate_sets:
+ for example in candidate_set:
+ if "augmented" in example.keys():
+ fields_to_use = get_signature(program.predictors()[pred_i]).fields
+ yield create_example_string(fields_to_use, example)
+ count += 1
+ if count >= max_examples:
+ return
+
+ # Construct full program demo or single module demo depending on settings
basic_instruction = get_signature(program.predictors()[pred_i]).instructions
- curr_demos_num = 0
+ task_demos = ""
if self.use_task_demos:
- for example in demo_candidates[pred_i][demo_set_i]:
- if "augmented" in example.keys():
- fields_to_use = get_signature(program.predictors()[pred_i]).fields
- example_string = create_example_string(fields_to_use, example)
- task_demos += f"{example_string}\n"
- curr_demos_num += 1
- if curr_demos_num >= max_demos:
- break
- else:
+ # Combine current and adjacent sets
+ adjacent_sets = (
+ [demo_candidates[pred_i][demo_set_i]] +
+ demo_candidates[pred_i][demo_set_i + 1:] +
+ demo_candidates[pred_i][:demo_set_i]
+ )
+
+ # Gather examples up to the required count
+ example_strings = gather_examples_from_sets(adjacent_sets, num_demos_in_context)
+ task_demos = "\n\n".join(example_strings) + "\n\n"
+
+ # Default to no demos provided if no examples were gathered, or if we're using the first demo set
+ if not task_demos.strip() or demo_set_i == 0:
task_demos = "No task demos provided."
# Summarize the program
@@ -224,20 +242,19 @@ class GenerateModuleInstruction(dspy.Module):
# Generate an instruction for our chosen module
if self.verbose: print(f"task_demos {task_demos}")
+
instruct = self.generate_module_instruction(
dataset_description=data_summary,
program_code=self.program_code_string,
- program_description=program_description,
module=module_code,
+ program_description=program_description,
+ module_description=module_description,
task_demos=task_demos,
tip=tip,
basic_instruction=basic_instruction,
previous_instructions=previous_instructions,
- module_description=module_description,
)
- if hasattr(instruct, "module_description"):
- module_description = strip_prefix(instruct.module_description)
- if self.verbose: print(f"MODULE DESCRIPTION: {module_description}")
+
proposed_instruction = strip_prefix(instruct.proposed_instruction)
return dspy.Prediction(proposed_instruction=proposed_instruction)
@@ -254,6 +271,7 @@ class GroundedProposer(Proposer):
use_dataset_summary=True,
program_aware=True,
use_task_demos=True,
+ num_demos_in_context = 3,
use_instruct_history=True,
use_tip=True,
set_tip_randomly=True,
@@ -265,6 +283,7 @@ class GroundedProposer(Proposer):
self.program_aware = program_aware
self.use_dataset_summary = use_dataset_summary
self.use_task_demos = use_task_demos
+ self.num_demos_in_context = num_demos_in_context
self.use_instruct_history = use_instruct_history
self.use_tip = use_tip
self.set_tip_randomly=set_tip_randomly
@@ -315,15 +334,13 @@ class GroundedProposer(Proposer):
self.use_instruct_history = use_history
if self.verbose: print(f"Use history T/F: {self.use_instruct_history}")
- num_demos = max(len(demo_candidates[0]) if demo_candidates else N, 1)
-
if not demo_candidates:
if self.verbose: print("No demo candidates provided. Running without task demos.")
self.use_task_demos = False
# Create an instruction for each predictor
for pred_i, predictor in enumerate(program.predictors()):
- for demo_set_i in range(num_demos):
+ for demo_set_i in range(len(demo_candidates[0])):
if pred_i not in proposed_instructions:
proposed_instructions[pred_i] = []
if self.set_tip_randomly:
@@ -341,7 +358,6 @@ class GroundedProposer(Proposer):
program=program,
predictor=predictor,
pred_i=pred_i,
- # prompt_model=prompt_model,
T=T,
demo_candidates=demo_candidates,
demo_set_i=demo_set_i,
@@ -349,7 +365,7 @@ class GroundedProposer(Proposer):
tip=selected_tip,
),
)
-
+
return proposed_instructions
def propose_instruction_for_predictor(
@@ -385,8 +401,11 @@ class GroundedProposer(Proposer):
# Generate a new instruction for our predictor, using the temperature specified for this round
original_temp = self.prompt_model.kwargs["temperature"]
+ epsilon = self.rng.uniform(0.01, 0.05)
+ modified_temp = T + epsilon
+
with dspy.settings.context(lm=self.prompt_model):
- self.prompt_model.kwargs["temperature"] = T
+ self.prompt_model.kwargs["temperature"] = modified_temp
proposed_instruction = instruction_generator.forward(
demo_candidates=demo_candidates,
pred_i=pred_i,
@@ -394,6 +413,7 @@ class GroundedProposer(Proposer):
program=program,
data_summary=self.data_summary,
previous_instructions=instruction_history,
+ num_demos_in_context = self.num_demos_in_context,
tip=tip,
).proposed_instruction
self.prompt_model.kwargs["temperature"] = original_temp
@@ -402,4 +422,4 @@ class GroundedProposer(Proposer):
if self.verbose: self.prompt_model.inspect_history(n=1)
if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")
- return strip_prefix(proposed_instruction)
+ return strip_prefix(proposed_instruction)
\ No newline at end of file
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index 5ac7f1dff..69314331d 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -437,6 +437,7 @@ class MIPROv2(Teleprompter):
program_aware=program_aware_proposer,
use_dataset_summary=data_aware_proposer,
use_task_demos=fewshot_aware_proposer,
+ num_demos_in_context = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT,
use_tip=tip_aware_proposer,
set_tip_randomly=tip_aware_proposer,
use_instruct_history=False,