DSPy: The framework for programming—not prompting—language models
Python
Switch new MIPRO to MIPROv2
Michael Ryanea1b32831e83
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.pyindex 39462634d..0105c1ebb 100644--- a/dspy/teleprompt/__init__.py+++ b/dspy/teleprompt/__init__.py@@ -4,6 +4,7 @@ from .ensemble import *from .finetune import *from .knn_fewshot import *from .mipro_optimizer import MIPRO+from .mipro_optimizer_v2 import MIPROv2from .random_search import *from .signature_opt import SignatureOptimizerfrom .signature_opt_bayesian import BayesianSignatureOptimizerdiff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.pyindex b9265a279..9f3c7c3cd 100644--- a/dspy/teleprompt/mipro_optimizer.py+++ b/dspy/teleprompt/mipro_optimizer.py@@ -1,35 +1,19 @@-import logging-import os-import pickle+import mathimport randomimport sysimport textwrapfrom collections import defaultdict+from typing import Any-import numpy as npimport optuna+import dsp+import dspyfrom dspy.evaluate.evaluate import Evaluate-from dspy.propose import GroundedProposer+from dspy.signatures import Signature+from dspy.signatures.signature import signature_to_template+from dspy.teleprompt import BootstrapFewShotfrom dspy.teleprompt.teleprompt import Teleprompter-from dspy.teleprompt.utils import (- create_n_fewshot_demo_sets,- eval_candidate_program,- get_dspy_source_code,- get_program_with_highest_avg_score,- get_signature,- get_task_model_history_for_full_example,- print_full_program,- save_candidate_program,- save_file_to_log_dir,- set_signature,- setup_logging,-)--try:- import wandb-except ImportError:- wandb = None"""USAGE SUGGESTIONS:@@ -41,7 +25,7 @@ from dspy.teleprompt import MIPROteleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_batches=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)```@@ -49,7 +33,6 @@ Note that this teleprompter takes in the following parameters:* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).-* teacher_settings: The settings used for the teacher model. When unspecified, defaults to the settings set in settings (i.e., dspy.settings.configure(lm=task_model)).* metric: The task metric used for optimization.* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.@@ -61,78 +44,324 @@ Note that this teleprompter takes in the following parameters:* score: the last average evaluated score for the program* pruned: whether or not this program was prunedThis information will be returned as attributes of the best program.-* log_dir: The directory to save logs and other information to. If unspecified, no logs will be saved.-* view_data_batch_size: The number of examples to view in the data batch when producing the dataset summary. Default=10.-* minibatch_size: The size of the minibatch to use when evaluating the program if using minibatched evaluations. Default=25.-* minibatch_full_eval_steps: The number of steps to take before doing a full evaluation of the program if using minibatched evaluations. Default=10."""-BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3-LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0-MB_FULL_EVAL_STEPS = 10-MINIBATCH_SIZE = 25#50+class BasicGenerateInstruction(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-class MIPRO(Teleprompter):+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithDataObservations(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ observations = dspy.InputField(desc="Observations about the dataset and task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithExamples(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.++ Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.++ Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ observations = dspy.InputField(desc="Observations about the dataset and task")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class ObservationSummarizer(dspy.Signature):+ """Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."""++ observations = dspy.InputField(desc="Observations I have made about my dataset")+ summary = dspy.OutputField(+ desc="Two to Three sentence summary of only the most significant highlights of my observations",+ )+++class DatasetDescriptor(dspy.Signature):+ (+ """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""+ )++ examples = dspy.InputField(desc="Sample data points from the dataset")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")+++class DatasetDescriptorWithPriorObservations(dspy.Signature):+ (+ """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""+ )++ examples = dspy.InputField(desc="Sample data points from the dataset")+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")+ observations = dspy.OutputField(+ desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add",+ )+++class MIPRO_DEPRECATED(Teleprompter):def __init__(self,+ metric,prompt_model=None,task_model=None,teacher_settings={},num_candidates=10,- metric=None,- init_temperature=1.4,+ init_temperature=1.0,verbose=False,track_stats=True,- log_dir=None,view_data_batch_size=10,- minibatch_size=MINIBATCH_SIZE,- minibatch_full_eval_steps=MB_FULL_EVAL_STEPS,):- self.n = num_candidates+ self.num_candidates = num_candidatesself.metric = metricself.init_temperature = init_temperature- self.prompt_model = prompt_model- self.task_model = task_model+ self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm+ self.task_model = task_model if task_model is not None else dspy.settings.lmself.verbose = verboseself.track_stats = track_stats- self.log_dir = log_dir- self.view_data_batch_size = view_data_batch_sizeself.teacher_settings = teacher_settings- self.prompt_model_total_calls = 0- self.total_calls = 0- self.minibatch_size = minibatch_size- self.minibatch_full_eval_steps = minibatch_full_eval_steps-- # Check if WANDB_RUN_ID is set in the environment- self.wandb_run_id = None-- def _get_batch_size(+ self.view_data_batch_size = view_data_batch_size++ def _print_full_program(self, program):+ for i, predictor in enumerate(program.predictors()):+ if self.verbose:+ print(f"Predictor {i}")+ if self.verbose:+ print(f"i: {self._get_signature(predictor).instructions}")+ *_, last_field = self._get_signature(predictor).fields.values()+ if self.verbose:+ print(f"p: {last_field.json_schema_extra['prefix']}")+ if self.verbose:+ print("\n")++ def _print_model_history(self, model, n=1):+ if self.verbose:+ print(f"Model ({model}) History:")+ model.inspect_history(n=n)++ def _observe_data(self, trainset, max_iterations=10):+ upper_lim = min(len(trainset), self.view_data_batch_size)+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))+ observations = observation["observations"]++ skips = 0+ iterations = 0+ for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):+ upper_lim = min(len(trainset), b + self.view_data_batch_size)+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(+ prior_observations=observations,+ examples=(trainset[b:upper_lim].__repr__()),+ )+ iterations += 1+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":+ skips += 1+ if skips >= 5:+ break+ continue+ if iterations >= max_iterations:+ break+ observations += output["observations"]++ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)++ return summary.summary++ def _create_example_string(self, fields, example):+ # Building the output string+ output = []+ for field in fields:+ name = field.name+ separator = field.separator+ input_variable = field.input_variable++ # Determine the value from input_data or prediction_data+ value = example.get(input_variable)++ # Construct the string for the current field+ field_str = f"{name}{separator}{value}"+ output.append(field_str)++ # Joining all the field strings+ return "\n".join(output)++ def _get_signature(self, predictor):+ if hasattr(predictor, "extended_signature"):+ return predictor.extended_signature+ elif hasattr(predictor, "signature"):+ return predictor.signature+ return None++ def _set_signature(self, predictor, updated_signature):+ if hasattr(predictor, "extended_signature"):+ predictor.extended_signature = updated_signature+ elif hasattr(predictor, "signature"):+ predictor.signature = updated_signature++ def _generate_first_N_candidates( # noqa: N802self,- minibatch,- trainset,- ):- if minibatch:- return self.minibatch_size- else:- return len(trainset)+ module: dspy.Module,+ N: int, # noqa: N803+ view_data: bool,+ view_examples: bool,+ demo_candidates: dict,+ devset,+ ) -> tuple[dict, dict]:+ candidates = {}+ evaluated_candidates = defaultdict(dict)++ if view_data:+ # Create data observations+ self.observations = None+ with dspy.settings.context(lm=self.prompt_model):+ self.observations = self._observe_data(devset).replace("Observations:", "").replace("Summary:", "")++ if view_examples:+ example_sets = {}+ for predictor in module.predictors():+ # Get all augmented examples+ example_set = {}+ all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples+ for example_set_i, set_of_examples in enumerate(all_sets_of_examples):+ if example_set_i != 0: # Skip the no examples case+ for example in set_of_examples: # Get each individual example in the set+ if "augmented" in example and example["augmented"]:+ if example_set_i not in example_set:+ example_set[example_set_i] = []+ fields_to_use = signature_to_template(predictor.signature).fields+ _input_variable_names = list(self._get_signature(predictor).input_fields.keys())+ example_string = self._create_example_string(fields_to_use, example)+ example_set[example_set_i].append(example_string)+ example_sets[id(predictor)] = example_set+ else:+ example_set[example_set_i] = []+ example_sets[id(predictor)] = example_set++ # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts+ for predictor in module.predictors():+ basic_instruction = None+ basic_prefix = None+ basic_instruction = self._get_signature(predictor).instructions+ *_, last_field = self._get_signature(predictor).fields.values()+ basic_prefix = last_field.json_schema_extra["prefix"]+ with dspy.settings.context(lm=self.prompt_model):+ # Data & Examples+ if view_data and view_examples:+ if 1 not in example_sets[id(predictor)]:+ raise ValueError("No examples found for the given predictor")+ instruct = None+ for i in range(1, self.num_candidates):+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamplesAndDataObservations,+ n=1,+ temperature=self.init_temperature,+ )(+ basic_instruction=basic_instruction,+ observations=self.observations,+ examples=example_sets[id(predictor)][i],+ )+ if not instruct:+ instruct = new_instruct+ else:+ instruct.completions.proposed_instruction.extend(+ new_instruct.completions.proposed_instruction,+ )+ instruct.completions.proposed_prefix_for_output_field.extend(+ new_instruct.completions.proposed_prefix_for_output_field,+ )+ # Just data+ elif view_data:+ instruct = dspy.Predict(+ BasicGenerateInstructionWithDataObservations,+ n=N - 1,+ temperature=self.init_temperature,+ )(basic_instruction=basic_instruction, observations=self.observations)+ # Just examples+ elif view_examples:+ instruct = None+ for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamples,+ n=1,+ temperature=self.init_temperature,+ )(+ basic_instruction=basic_instruction,+ examples=example_sets[id(predictor)][i],+ )+ if not instruct:+ instruct = new_instruct+ else:+ instruct.completions.proposed_instruction.extend(+ new_instruct.completions.proposed_instruction,+ )+ instruct.completions.proposed_prefix_for_output_field.extend(+ new_instruct.completions.proposed_prefix_for_output_field,+ )+ # Neither+ else:+ instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)(+ basic_instruction=basic_instruction,+ )++ # Add in our initial prompt as a candidate as well+ instruct.completions.proposed_instruction.insert(0, basic_instruction)+ instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)+ candidates[id(predictor)] = instruct.completions+ evaluated_candidates[id(predictor)] = {}++ if self.verbose:+ self._print_model_history(self.prompt_model)++ return candidates, evaluated_candidatesdef compile(self,- student,+ student: dspy.Program,*,- trainset,- valset=[],- num_batches=30,- max_bootstrapped_demos=5,- max_labeled_demos=2,- eval_kwargs={},- seed=9,- minibatch=True,- program_aware_proposer=True,+ trainset: list[dspy.Example],+ num_trials: int,+ max_bootstrapped_demos: int,+ max_labeled_demos: int,+ eval_kwargs: dict[str, Any],+ seed=42,+ view_data=True,+ view_examples=True,requires_permission_to_run=True,- ):+ ) -> dspy.Program:# Define ANSI escape codes for colors+ RED = "\033[91m"YELLOW = "\033[93m"BLUE = "\033[94m"BOLD = "\033[1m"@@ -140,34 +369,18 @@ class MIPRO(Teleprompter):random.seed(seed)- estimated_prompt_model_calls = 10 + self.n * len(- student.predictors(),- ) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)-- prompt_model_line = ""- if not program_aware_proposer:- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""- else:- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""-- estimated_task_model_calls_wo_module_calls = 0- task_model_line = ""- if not minibatch:- estimated_task_model_calls_wo_module_calls = len(trainset) * num_batches # M * T * P- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""- else:- estimated_task_model_calls_wo_module_calls = self.minibatch_size * num_batches + (len(trainset) * (num_batches // self.minibatch_full_eval_steps)) # B * T * P- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{self.minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches // self.minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""-+ estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P+ estimated_prompt_model_calls = 10 + self.num_candidates * len(+ student.predictors(),+ ) # num data summary calls + N * Puser_message = textwrap.dedent(f"""\{YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:-- {prompt_model_line}- {task_model_line}+ {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}+ {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}{YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}@@ -177,7 +390,7 @@ class MIPRO(Teleprompter):For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the taskand prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:- {YELLOW}- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.{ENDC}+ {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}{YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")user_confirmation_message = textwrap.dedent(f"""\@@ -188,6 +401,7 @@ class MIPRO(Teleprompter):{YELLOW}Awaiting your input...{ENDC}""")+ print(f"""{RED}{BOLD}WARNING: MIPRO has been deprecated and replaced with MIPROv2. MIPRO will be removed in a future release. {ENDC}""")print(user_message)sys.stdout.flush() # Flush the output buffer to force the message to print@@ -201,308 +415,186 @@ class MIPRO(Teleprompter):run = Falseif run:- if program_aware_proposer:- try:- self.program_code_string = get_dspy_source_code(student)- if self.verbose:- print("SOURCE CODE:",self.program_code_string)- except Exception as e:- print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")- self.program_code_string = None- program_aware_proposer = False-- # Setup our proposer- proposer = GroundedProposer(- trainset=trainset,- prompt_model=self.prompt_model,- program_code_string=self.program_code_string,- program_aware=program_aware_proposer,- )-- # Setup logging- logging.basicConfig(level=logging.WARNING)- setup_logging(self.log_dir)-- self.wandb_run_id = os.getenv("WANDB_RUN_ID", None)-- if self.wandb_run_id:- # Initialize wandb with the same run ID- wandb.init(- project="prompt_optimizers",- id=self.wandb_run_id,- resume="must",- )-- # Setup random seeds- random.seed(seed)- np.random.seed(seed)-- # Log current file to log_dir- curr_file = os.path.abspath(__file__)- save_file_to_log_dir(curr_file, self.log_dir)-# Set up program and evaluation function- program = student.deepcopy()+ module = student.deepcopy()evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)- # Determine the number of fewshot examples to use to generate demos for prompt- if max_bootstrapped_demos == 0 and max_labeled_demos == 0:- max_bootstrapped_demos_for_candidate_gen = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT- max_labeled_demos_for_candidate_gen = LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT+ # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt+ if (+ max_bootstrapped_demos == 0 and max_labeled_demos == 0+ ): # TODO: address case when max_bootstrapped alone is 0+ max_bootstrapped_demos_for_candidate_gen = 1+ max_labeled_demos_for_candidate_gen = 1 # TODO: this might only need to be 0else:max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demosmax_labeled_demos_for_candidate_gen = max_labeled_demos- # # Generate N few shot example sets- # if not demo_candidates and not instruction_candidates:- # demo_candidates = create_n_fewshot_example_sets(program=program, trainset=trainset, n=self.n, hard_fewshot=hard_fewshot, metric=self.metric, teacher_settings=self.teacher_settings, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen)-# Generate N few shot example sets- try:- demo_candidates = create_n_fewshot_demo_sets(- student=program,- num_candidate_sets=self.n,- trainset=trainset,- max_labeled_demos=max_labeled_demos_for_candidate_gen,- max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,- metric=self.metric,- teacher_settings=self.teacher_settings,- seed=seed,- )- except Exception as e:- print(f"Error generating fewshot examples: {e}")- print("Running without fewshot examples.")- demo_candidates = None+ demo_candidates = {}+ for i in range(self.num_candidates):+ if i == 0: # Story empty set of demos as default for index 0+ for module_p in module.predictors():+ if id(module_p) not in demo_candidates:+ demo_candidates[id(module_p)] = []+ demo_candidates[id(module_p)].append([])+ else:+ if self.verbose:+ print(f"Creating basic bootstrap: {i}/{self.num_candidates-1}")++ # Create a new basic bootstrap few - shot program .+ rng = random.Random(i)+ shuffled_trainset = trainset[:] # Create a copy of devset+ rng.shuffle(shuffled_trainset) # Shuffle the copy+ tp = BootstrapFewShot(+ metric=self.metric,+ max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,+ max_labeled_demos=max_labeled_demos_for_candidate_gen,+ teacher_settings=self.teacher_settings,+ )+ candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_trainset)++ # Store the candidate demos+ for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):+ if id(module_p) not in demo_candidates:+ demo_candidates[id(module_p)] = []+ demo_candidates[id(module_p)].append(candidate_p.demos)# Generate N candidate prompts- proposer.program_aware = program_aware_proposer- proposer.use_tip = True- proposer.use_instruct_history = False- proposer.set_history_randomly = False- instruction_candidates = proposer.propose_instructions_for_program(- trainset=trainset,- program=program,- demo_candidates=demo_candidates,- N=self.n,- prompt_model=self.prompt_model,- T=self.init_temperature,- trial_logs={},+ instruction_candidates, _ = self._generate_first_N_candidates(+ module,+ self.num_candidates,+ view_data,+ view_examples,+ demo_candidates,+ trainset,)- for i, pred in enumerate(program.predictors()):- instruction_candidates[i][0] = get_signature(pred).instructions- # instruction_candidates[1][0] = "Given the question, and context, respond with the number of the document that is most relevant to answering the question in the field 'Answer' (ex. Answer: '3')."-- # Save the candidate instructions generated- if self.log_dir:- fp = os.path.join(self.log_dir, "instructions_to_save.pickle")- with open(fp, "wb") as file:- pickle.dump(instruction_candidates, file)-- # If we're doing zero-shot, reset demo_candidates to none+ # Reset demo_candidates to None for our optimization if the user asked for no fewshot examplesif max_bootstrapped_demos == 0 and max_labeled_demos == 0:demo_candidates = None# Initialize variables to store the best program and its scorebest_score = float("-inf")best_program = None+ trial_num = 0+trial_logs = {}- total_eval_calls = 0- param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scores- fully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of# Define our trial objective- def create_objective(- baseline_program,- instruction_candidates,- demo_candidates,- evaluate,- trainset,- ):+ def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, trainset):def objective(trial):- nonlocal best_program, best_score, trial_logs, total_eval_calls # Allow access to the outer variables-- # Kick off trial- logging.info(f"Starting trial num: {trial.number}")- trial_logs[trial.number] = {}-- # Create a new candidate program+ nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variablescandidate_program = baseline_program.deepcopy()- # Choose set of instructions & demos to use for each predictor- chosen_params = []- for i, p_new in enumerate(candidate_program.predictors()):+ # Suggest the instruction to use for our predictor+ print(f"Starting trial #{trial_num}")+ trial_logs[trial_num] = {}- # Get instruction candidates / demos for our given predictor- p_instruction_candidates = instruction_candidates[i]+ for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):+ # Get instruction candidates for our given predictor+ p_instruction_candidates = instruction_candidates[id(p_old)]if demo_candidates:- p_demo_candidates = demo_candidates[i]+ p_demo_candidates = demo_candidates[id(p_old)]- # Suggest the index of the instruction / demo candidate to use in our trial+ # Suggest the index of the instruction candidate to use in our trialinstruction_idx = trial.suggest_categorical(- f"{i}_predictor_instruction",+ f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)),)- chosen_params.append(instruction_idx)if demo_candidates:demos_idx = trial.suggest_categorical(- f"{i}_predictor_demos", range(len(p_demo_candidates)),+ f"{id(p_old)}_predictor_demos",+ range(len(p_demo_candidates)),)- chosen_params.append(demo_candidates)-- # Log the selected instruction / demo candidate- trial_logs[trial.number][- f"{i}_predictor_instruction"- ] = instruction_idx+ trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idxif demo_candidates:- trial_logs[trial.number][f"{i}_predictor_demos"] = demos_idx-- logging.info(f"instruction_idx {instruction_idx}")- if demo_candidates:- logging.info(f"demos_idx {demos_idx}")-- # Set the instruction- selected_instruction = p_instruction_candidates[instruction_idx]- updated_signature = get_signature(p_new).with_instructions(- selected_instruction,+ trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx++ # Get the selected instruction candidate+ selected_candidate = p_instruction_candidates[instruction_idx]+ selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()+ selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()++ # Use this candidates in our program+ *_, last_field = self._get_signature(p_new).fields.keys()+ updated_signature = (+ self._get_signature(p_new)+ .with_instructions(selected_instruction)+ .with_updated_fields(last_field, prefix=selected_prefix))- set_signature(p_new, updated_signature)+ self._set_signature(p_new, updated_signature)- # Set the demos+ # Get the selected demosif demo_candidates:- p_new.demos = p_demo_candidates[demos_idx]+ selected_demos = p_demo_candidates[demos_idx]- # Log assembled program- print("CANDIDATE PROGRAM:")- print_full_program(candidate_program)- print("...")-- # Save the candidate program- trial_logs[trial.number]["program_path"] = save_candidate_program(- candidate_program, self.log_dir, trial.number,- )+ # Use these demos in our program+ if demo_candidates:+ p_new.demos = selected_demos- trial_logs[trial.number]["num_eval_calls"] = 0+ if self.verbose:+ print("Evaling the following program:")+ if self.verbose:+ self._print_full_program(candidate_program)+ trial_logs[trial_num]["program"] = candidate_program++ # Evaluate with the new prompts+ total_score = 0+ batch_size = 100+ num_batches = math.ceil(len(trainset) / batch_size)++ for i in range(num_batches):+ start_index = i * batch_size+ end_index = min((i + 1) * batch_size, len(trainset))+ split_trainset = trainset[start_index:end_index]+ split_score = evaluate(candidate_program, devset=split_trainset, display_table=0)+ if self.verbose:+ print(f"{i}st split score: {split_score}")++ total_score += split_score * len(split_trainset)+ curr_weighted_avg_score = total_score / min((i + 1) * 100, len(trainset))+ if self.verbose:+ print(f"curr average score: {curr_weighted_avg_score}")++ trial.report(curr_weighted_avg_score, i)++ # Handle pruning based on the intermediate value.+ if trial.should_prune():+ print("Trial pruned.")+ trial_logs[trial_num]["score"] = curr_weighted_avg_score+ trial_logs[trial_num]["pruned"] = True+ trial_num += 1+ raise optuna.TrialPruned()- # Evaluate the candidate program with relevant batch size- batch_size = self._get_batch_size(minibatch, trainset)- score = eval_candidate_program(- batch_size, trainset, candidate_program, evaluate,- )+ if self.verbose:+ print(f"Fully evaled score: {curr_weighted_avg_score}")+ if self.verbose:+ self._print_model_history(self.task_model, n=1)+ score = curr_weighted_avg_score- # Print out a full trace of the program in use- print("FULL TRACE")- full_trace = get_task_model_history_for_full_example(- candidate_program, self.task_model, trainset, evaluate,- )- print("...")+ trial_logs[trial_num]["score"] = curr_weighted_avg_score+ trial_logs[trial_num]["pruned"] = False- # Log relevant information- print(f"Score {score}")- categorical_key = ",".join(map(str, chosen_params))- param_score_dict[categorical_key].append(- (score, candidate_program),- )- trial_logs[trial.number]["num_eval_calls"] = batch_size- trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)- trial_logs[trial.number]["eval_example_call"] = full_trace- trial_logs[trial.number]["score"] = score- trial_logs[trial.number]["pruned"] = False- total_eval_calls += trial_logs[trial.number]["num_eval_calls"]- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number]["program"] = candidate_program.deepcopy()- if self.wandb_run_id:- wandb.log(- {- "score": score,- "num_eval_calls": trial_logs[trial.number]["num_eval_calls"],- "total_eval_calls": total_eval_calls,- },- )-- # Update the best program if the current score is better, and if we're not using minibatching- best_score_updated = False- if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:- print("Updating best score")+ # Update the best program if the current score is better+ if score > best_score:best_score = scorebest_program = candidate_program.deepcopy()- best_score_updated = True--- # If we're doing minibatching, check to see if it's time to do a full eval- if minibatch and trial.number % self.minibatch_full_eval_steps == 0:-- # Save old information as the minibatch version- trial_logs[trial.number]["mb_score"] = score- trial_logs[trial.number]["mb_program_path"] = trial_logs[trial.number]["program_path"]-- # Identify our best program (based on mean of scores so far, and do a full eval on it)- highest_mean_program, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)- full_train_score = eval_candidate_program(- len(trainset), trainset, highest_mean_program, evaluate,- )- # Log relevant information- fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}- total_eval_calls += len(trainset)- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number]["full_eval"] = True- trial_logs[trial.number]["program_path"] = save_candidate_program(- program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number, note="full_eval",- )- trial_logs[trial.number]["score"] = full_train_score-- if full_train_score > best_score:- print(f"UPDATING BEST SCORE WITH {full_train_score}")- best_score = full_train_score- best_program = highest_mean_program.deepcopy()- best_score_updated = True-- # If the best score was updated, do a full eval on the dev set- if best_score_updated:- full_dev_score = evaluate(- best_program,- devset=valset,- display_table=0,- )- if self.wandb_run_id:- wandb.log(- {- "best_prog_so_far_train_score": best_score,- "best_prog_so_far_dev_score": full_dev_score,- },- )+ trial_num += 1return scorereturn objective# Run the trial- objective_function = create_objective(- program, instruction_candidates, demo_candidates, evaluate, trainset,- )-- # if minibatch:- # num_batches = int(num_batches*len(trainset)/(self.minibatch_size + len(trainset)/self.minibatch_full_eval_steps))-- sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)+ objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)+ sampler = optuna.samplers.TPESampler(seed=seed)study = optuna.create_study(direction="maximize", sampler=sampler)- score = study.optimize(objective_function, n_trials=num_batches)+ _score = study.optimize(objective_function, n_trials=num_trials)if best_program is not None and self.track_stats:best_program.trial_logs = trial_logs- best_program.score = best_score- best_program.prompt_model_total_calls = self.prompt_model_total_calls- best_program.total_calls = self.total_calls-- # program_file_path = os.path.join(self.log_dir, 'best_program.pickle')- if self.log_dir:- program_file_path = os.path.join(self.log_dir, "best_program")- best_program.save(program_file_path)-- optuna_study_file_path = os.path.join(self.log_dir, "optuna_study.pickle")- with open(optuna_study_file_path, "wb") as file:- pickle.dump(study, file)+ print(f"Returning {best_program} from continue_program")return best_program- return student+ return Nonediff --git a/dspy/teleprompt/mipro_optimizer_deprecated.py b/dspy/teleprompt/mipro_optimizer_deprecated.pydeleted file mode 100644index 034ea1ef3..000000000--- a/dspy/teleprompt/mipro_optimizer_deprecated.py+++ /dev/null@@ -1,598 +0,0 @@-import math-import random-import sys-import textwrap-from collections import defaultdict-from typing import Any--import optuna--import dsp-import dspy-from dspy.evaluate.evaluate import Evaluate-from dspy.signatures import Signature-from dspy.signatures.signature import signature_to_template-from dspy.teleprompt import BootstrapFewShot-from dspy.teleprompt.teleprompt import Teleprompter--"""-USAGE SUGGESTIONS:--The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:--``` python-from dspy.teleprompt import MIPRO--teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)-kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)-eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)-```--Note that this teleprompter takes in the following parameters:--* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).-* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).-* metric: The task metric used for optimization.-* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.-* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.-* verbose: Tells the method whether or not to print intermediate steps.-* track_stats: Tells the method whether or not to track statistics about the optimization process.- If True, the method will track a dictionary with a key corresponding to the trial number,- and a value containing a dict with the following keys:- * program: the program being evaluated at a given trial- * score: the last average evaluated score for the program- * pruned: whether or not this program was pruned- This information will be returned as attributes of the best program.-"""---class BasicGenerateInstruction(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithDataObservations(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- observations = dspy.InputField(desc="Observations about the dataset and task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithExamples(dspy.Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.-- Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.-- Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- observations = dspy.InputField(desc="Observations about the dataset and task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class ObservationSummarizer(dspy.Signature):- """Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."""-- observations = dspy.InputField(desc="Observations I have made about my dataset")- summary = dspy.OutputField(- desc="Two to Three sentence summary of only the most significant highlights of my observations",- )---class DatasetDescriptor(dspy.Signature):- (- """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""- )-- examples = dspy.InputField(desc="Sample data points from the dataset")- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")---class DatasetDescriptorWithPriorObservations(dspy.Signature):- (- """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""- )-- examples = dspy.InputField(desc="Sample data points from the dataset")- prior_observations = dspy.InputField(desc="Some prior observations I made about the data")- observations = dspy.OutputField(- desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add",- )---class MIPRO_DEPRECATED(Teleprompter):- def __init__(- self,- metric,- prompt_model=None,- task_model=None,- teacher_settings={},- num_candidates=10,- init_temperature=1.0,- verbose=False,- track_stats=True,- view_data_batch_size=10,- ):- self.num_candidates = num_candidates- self.metric = metric- self.init_temperature = init_temperature- self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm- self.task_model = task_model if task_model is not None else dspy.settings.lm- self.verbose = verbose- self.track_stats = track_stats- self.teacher_settings = teacher_settings- self.view_data_batch_size = view_data_batch_size-- def _print_full_program(self, program):- for i, predictor in enumerate(program.predictors()):- if self.verbose:- print(f"Predictor {i}")- if self.verbose:- print(f"i: {self._get_signature(predictor).instructions}")- *_, last_field = self._get_signature(predictor).fields.values()- if self.verbose:- print(f"p: {last_field.json_schema_extra['prefix']}")- if self.verbose:- print("\n")-- def _print_model_history(self, model, n=1):- if self.verbose:- print(f"Model ({model}) History:")- model.inspect_history(n=n)-- def _observe_data(self, trainset, max_iterations=10):- upper_lim = min(len(trainset), self.view_data_batch_size)- observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))- observations = observation["observations"]-- skips = 0- iterations = 0- for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):- upper_lim = min(len(trainset), b + self.view_data_batch_size)- output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(- prior_observations=observations,- examples=(trainset[b:upper_lim].__repr__()),- )- iterations += 1- if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":- skips += 1- if skips >= 5:- break- continue- if iterations >= max_iterations:- break- observations += output["observations"]-- summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)-- return summary.summary-- def _create_example_string(self, fields, example):- # Building the output string- output = []- for field in fields:- name = field.name- separator = field.separator- input_variable = field.input_variable-- # Determine the value from input_data or prediction_data- value = example.get(input_variable)-- # Construct the string for the current field- field_str = f"{name}{separator}{value}"- output.append(field_str)-- # Joining all the field strings- return "\n".join(output)-- def _get_signature(self, predictor):- if hasattr(predictor, "extended_signature"):- return predictor.extended_signature- elif hasattr(predictor, "signature"):- return predictor.signature- return None-- def _set_signature(self, predictor, updated_signature):- if hasattr(predictor, "extended_signature"):- predictor.extended_signature = updated_signature- elif hasattr(predictor, "signature"):- predictor.signature = updated_signature-- def _generate_first_N_candidates( # noqa: N802- self,- module: dspy.Module,- N: int, # noqa: N803- view_data: bool,- view_examples: bool,- demo_candidates: dict,- devset,- ) -> tuple[dict, dict]:- candidates = {}- evaluated_candidates = defaultdict(dict)-- if view_data:- # Create data observations- self.observations = None- with dspy.settings.context(lm=self.prompt_model):- self.observations = self._observe_data(devset).replace("Observations:", "").replace("Summary:", "")-- if view_examples:- example_sets = {}- for predictor in module.predictors():- # Get all augmented examples- example_set = {}- all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples- for example_set_i, set_of_examples in enumerate(all_sets_of_examples):- if example_set_i != 0: # Skip the no examples case- for example in set_of_examples: # Get each individual example in the set- if "augmented" in example and example["augmented"]:- if example_set_i not in example_set:- example_set[example_set_i] = []- fields_to_use = signature_to_template(predictor.signature).fields- _input_variable_names = list(self._get_signature(predictor).input_fields.keys())- example_string = self._create_example_string(fields_to_use, example)- example_set[example_set_i].append(example_string)- example_sets[id(predictor)] = example_set- else:- example_set[example_set_i] = []- example_sets[id(predictor)] = example_set-- # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts- for predictor in module.predictors():- basic_instruction = None- basic_prefix = None- basic_instruction = self._get_signature(predictor).instructions- *_, last_field = self._get_signature(predictor).fields.values()- basic_prefix = last_field.json_schema_extra["prefix"]- with dspy.settings.context(lm=self.prompt_model):- # Data & Examples- if view_data and view_examples:- if 1 not in example_sets[id(predictor)]:- raise ValueError("No examples found for the given predictor")- instruct = None- for i in range(1, self.num_candidates):- new_instruct = dspy.Predict(- BasicGenerateInstructionWithExamplesAndDataObservations,- n=1,- temperature=self.init_temperature,- )(- basic_instruction=basic_instruction,- observations=self.observations,- examples=example_sets[id(predictor)][i],- )- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(- new_instruct.completions.proposed_instruction,- )- instruct.completions.proposed_prefix_for_output_field.extend(- new_instruct.completions.proposed_prefix_for_output_field,- )- # Just data- elif view_data:- instruct = dspy.Predict(- BasicGenerateInstructionWithDataObservations,- n=N - 1,- temperature=self.init_temperature,- )(basic_instruction=basic_instruction, observations=self.observations)- # Just examples- elif view_examples:- instruct = None- for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty- new_instruct = dspy.Predict(- BasicGenerateInstructionWithExamples,- n=1,- temperature=self.init_temperature,- )(- basic_instruction=basic_instruction,- examples=example_sets[id(predictor)][i],- )- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(- new_instruct.completions.proposed_instruction,- )- instruct.completions.proposed_prefix_for_output_field.extend(- new_instruct.completions.proposed_prefix_for_output_field,- )- # Neither- else:- instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)(- basic_instruction=basic_instruction,- )-- # Add in our initial prompt as a candidate as well- instruct.completions.proposed_instruction.insert(0, basic_instruction)- instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)- candidates[id(predictor)] = instruct.completions- evaluated_candidates[id(predictor)] = {}-- if self.verbose:- self._print_model_history(self.prompt_model)-- return candidates, evaluated_candidates-- def compile(- self,- student: dspy.Program,- *,- trainset: list[dspy.Example],- num_trials: int,- max_bootstrapped_demos: int,- max_labeled_demos: int,- eval_kwargs: dict[str, Any],- seed=42,- view_data=True,- view_examples=True,- requires_permission_to_run=True,- ) -> dspy.Program:- # Define ANSI escape codes for colors- YELLOW = "\033[93m"- BLUE = "\033[94m"- BOLD = "\033[1m"- ENDC = "\033[0m" # Resets the color to default-- random.seed(seed)-- estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P- estimated_prompt_model_calls = 10 + self.num_candidates * len(- student.predictors(),- ) # num data summary calls + N * P-- user_message = textwrap.dedent(f"""\- {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}-- Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:-- {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}- {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}-- {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}-- {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token)- + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC}-- For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task- and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:-- {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}- {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")-- user_confirmation_message = textwrap.dedent(f"""\- To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.-- If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}-- {YELLOW}Awaiting your input...{ENDC}- """)-- print(user_message)-- sys.stdout.flush() # Flush the output buffer to force the message to print-- run = True- if requires_permission_to_run:- print(user_confirmation_message)- user_input = input("Do you wish to continue? (y/n): ").strip().lower()- if user_input != "y":- print("Compilation aborted by the user.")- run = False-- if run:- # Set up program and evaluation function- module = student.deepcopy()- evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)-- # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt- if (- max_bootstrapped_demos == 0 and max_labeled_demos == 0- ): # TODO: address case when max_bootstrapped alone is 0- max_bootstrapped_demos_for_candidate_gen = 1- max_labeled_demos_for_candidate_gen = 1 # TODO: this might only need to be 0- else:- max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos- max_labeled_demos_for_candidate_gen = max_labeled_demos-- # Generate N few shot example sets- demo_candidates = {}- for i in range(self.num_candidates):- if i == 0: # Story empty set of demos as default for index 0- for module_p in module.predictors():- if id(module_p) not in demo_candidates:- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append([])- else:- if self.verbose:- print(f"Creating basic bootstrap: {i}/{self.num_candidates-1}")-- # Create a new basic bootstrap few - shot program .- rng = random.Random(i)- shuffled_trainset = trainset[:] # Create a copy of devset- rng.shuffle(shuffled_trainset) # Shuffle the copy- tp = BootstrapFewShot(- metric=self.metric,- max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,- max_labeled_demos=max_labeled_demos_for_candidate_gen,- teacher_settings=self.teacher_settings,- )- candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_trainset)-- # Store the candidate demos- for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):- if id(module_p) not in demo_candidates:- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append(candidate_p.demos)-- # Generate N candidate prompts- instruction_candidates, _ = self._generate_first_N_candidates(- module,- self.num_candidates,- view_data,- view_examples,- demo_candidates,- trainset,- )-- # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples- if max_bootstrapped_demos == 0 and max_labeled_demos == 0:- demo_candidates = None-- # Initialize variables to store the best program and its score- best_score = float("-inf")- best_program = None- trial_num = 0-- trial_logs = {}-- # Define our trial objective- def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, trainset):- def objective(trial):- nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables- candidate_program = baseline_program.deepcopy()-- # Suggest the instruction to use for our predictor- print(f"Starting trial #{trial_num}")- trial_logs[trial_num] = {}-- for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):- # Get instruction candidates for our given predictor- p_instruction_candidates = instruction_candidates[id(p_old)]- if demo_candidates:- p_demo_candidates = demo_candidates[id(p_old)]-- # Suggest the index of the instruction candidate to use in our trial- instruction_idx = trial.suggest_categorical(- f"{id(p_old)}_predictor_instruction",- range(len(p_instruction_candidates)),- )- if demo_candidates:- demos_idx = trial.suggest_categorical(- f"{id(p_old)}_predictor_demos",- range(len(p_demo_candidates)),- )- trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx- if demo_candidates:- trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx-- # Get the selected instruction candidate- selected_candidate = p_instruction_candidates[instruction_idx]- selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()- selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()-- # Use this candidates in our program- *_, last_field = self._get_signature(p_new).fields.keys()- updated_signature = (- self._get_signature(p_new)- .with_instructions(selected_instruction)- .with_updated_fields(last_field, prefix=selected_prefix)- )- self._set_signature(p_new, updated_signature)-- # Get the selected demos- if demo_candidates:- selected_demos = p_demo_candidates[demos_idx]-- # Use these demos in our program- if demo_candidates:- p_new.demos = selected_demos-- if self.verbose:- print("Evaling the following program:")- if self.verbose:- self._print_full_program(candidate_program)- trial_logs[trial_num]["program"] = candidate_program-- # Evaluate with the new prompts- total_score = 0- batch_size = 100- num_batches = math.ceil(len(trainset) / batch_size)-- for i in range(num_batches):- start_index = i * batch_size- end_index = min((i + 1) * batch_size, len(trainset))- split_trainset = trainset[start_index:end_index]- split_score = evaluate(candidate_program, devset=split_trainset, display_table=0)- if self.verbose:- print(f"{i}st split score: {split_score}")-- total_score += split_score * len(split_trainset)- curr_weighted_avg_score = total_score / min((i + 1) * 100, len(trainset))- if self.verbose:- print(f"curr average score: {curr_weighted_avg_score}")-- trial.report(curr_weighted_avg_score, i)-- # Handle pruning based on the intermediate value.- if trial.should_prune():- print("Trial pruned.")- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = True- trial_num += 1- raise optuna.TrialPruned()-- if self.verbose:- print(f"Fully evaled score: {curr_weighted_avg_score}")- if self.verbose:- self._print_model_history(self.task_model, n=1)- score = curr_weighted_avg_score-- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = False-- # Update the best program if the current score is better- if score > best_score:- best_score = score- best_program = candidate_program.deepcopy()-- trial_num += 1-- return score-- return objective-- # Run the trial- objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)- sampler = optuna.samplers.TPESampler(seed=seed)- study = optuna.create_study(direction="maximize", sampler=sampler)- _score = study.optimize(objective_function, n_trials=num_trials)-- if best_program is not None and self.track_stats:- best_program.trial_logs = trial_logs-- print(f"Returning {best_program} from continue_program")- return best_program- return Nonediff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pynew file mode 100644index 000000000..97b5a9836--- /dev/null+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -0,0 +1,508 @@+import logging+import os+import pickle+import random+import sys+import textwrap+from collections import defaultdict++import numpy as np+import optuna++from dspy.evaluate.evaluate import Evaluate+from dspy.propose import GroundedProposer+from dspy.teleprompt.teleprompt import Teleprompter+from dspy.teleprompt.utils import (+ create_n_fewshot_demo_sets,+ eval_candidate_program,+ get_dspy_source_code,+ get_program_with_highest_avg_score,+ get_signature,+ get_task_model_history_for_full_example,+ print_full_program,+ save_candidate_program,+ save_file_to_log_dir,+ set_signature,+ setup_logging,+)++try:+ import wandb+except ImportError:+ wandb = None++"""+USAGE SUGGESTIONS:++The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:++``` python+from dspy.teleprompt import MIPROv2++teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)+kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_batches=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)+eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)+```++Note that this teleprompter takes in the following parameters:++* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* teacher_settings: The settings used for the teacher model. When unspecified, defaults to the settings set in settings (i.e., dspy.settings.configure(lm=task_model)).+* metric: The task metric used for optimization.+* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.+* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.+* verbose: Tells the method whether or not to print intermediate steps.+* track_stats: Tells the method whether or not to track statistics about the optimization process.+ If True, the method will track a dictionary with a key corresponding to the trial number,+ and a value containing a dict with the following keys:+ * program: the program being evaluated at a given trial+ * score: the last average evaluated score for the program+ * pruned: whether or not this program was pruned+ This information will be returned as attributes of the best program.+* log_dir: The directory to save logs and other information to. If unspecified, no logs will be saved.+* view_data_batch_size: The number of examples to view in the data batch when producing the dataset summary. Default=10.+* minibatch_size: The size of the minibatch to use when evaluating the program if using minibatched evaluations. Default=25.+* minibatch_full_eval_steps: The number of steps to take before doing a full evaluation of the program if using minibatched evaluations. Default=10.+"""++BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3+LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0++MB_FULL_EVAL_STEPS = 10+MINIBATCH_SIZE = 25#50++class MIPROv2(Teleprompter):+ def __init__(+ self,+ prompt_model=None,+ task_model=None,+ teacher_settings={},+ num_candidates=10,+ metric=None,+ init_temperature=1.4,+ verbose=False,+ track_stats=True,+ log_dir=None,+ view_data_batch_size=10,+ minibatch_size=MINIBATCH_SIZE,+ minibatch_full_eval_steps=MB_FULL_EVAL_STEPS,+ ):+ self.n = num_candidates+ self.metric = metric+ self.init_temperature = init_temperature+ self.prompt_model = prompt_model+ self.task_model = task_model+ self.verbose = verbose+ self.track_stats = track_stats+ self.log_dir = log_dir+ self.view_data_batch_size = view_data_batch_size+ self.teacher_settings = teacher_settings+ self.prompt_model_total_calls = 0+ self.total_calls = 0+ self.minibatch_size = minibatch_size+ self.minibatch_full_eval_steps = minibatch_full_eval_steps++ # Check if WANDB_RUN_ID is set in the environment+ self.wandb_run_id = None++ def _get_batch_size(+ self,+ minibatch,+ trainset,+ ):+ if minibatch:+ return self.minibatch_size+ else:+ return len(trainset)++ def compile(+ self,+ student,+ *,+ trainset,+ valset=[],+ num_batches=30,+ max_bootstrapped_demos=5,+ max_labeled_demos=2,+ eval_kwargs={},+ seed=9,+ minibatch=True,+ program_aware_proposer=True,+ requires_permission_to_run=True,+ ):+ # Define ANSI escape codes for colors+ YELLOW = "\033[93m"+ BLUE = "\033[94m"+ BOLD = "\033[1m"+ ENDC = "\033[0m" # Resets the color to default++ random.seed(seed)++ estimated_prompt_model_calls = 10 + self.n * len(+ student.predictors(),+ ) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)++ prompt_model_line = ""+ if not program_aware_proposer:+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""+ else:+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""++ estimated_task_model_calls_wo_module_calls = 0+ task_model_line = ""+ if not minibatch:+ estimated_task_model_calls_wo_module_calls = len(trainset) * num_batches # M * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""+ else:+ estimated_task_model_calls_wo_module_calls = self.minibatch_size * num_batches + (len(trainset) * (num_batches // self.minibatch_full_eval_steps)) # B * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{self.minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches // self.minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""+++ user_message = textwrap.dedent(f"""\+ {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}++ Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:+++ {prompt_model_line}+ {task_model_line}++ {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}++ {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token)+ + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC}++ For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task+ and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:++ {YELLOW}- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.{ENDC}+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")++ user_confirmation_message = textwrap.dedent(f"""\+ To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.++ If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}++ {YELLOW}Awaiting your input...{ENDC}+ """)++ print(user_message)++ sys.stdout.flush() # Flush the output buffer to force the message to print++ run = True+ if requires_permission_to_run:+ print(user_confirmation_message)+ user_input = input("Do you wish to continue? (y/n): ").strip().lower()+ if user_input != "y":+ print("Compilation aborted by the user.")+ run = False++ if run:+ if program_aware_proposer:+ try:+ self.program_code_string = get_dspy_source_code(student)+ if self.verbose:+ print("SOURCE CODE:",self.program_code_string)+ except Exception as e:+ print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")+ self.program_code_string = None+ program_aware_proposer = False++ # Setup our proposer+ proposer = GroundedProposer(+ trainset=trainset,+ prompt_model=self.prompt_model,+ program_code_string=self.program_code_string,+ program_aware=program_aware_proposer,+ )++ # Setup logging+ logging.basicConfig(level=logging.WARNING)+ setup_logging(self.log_dir)++ self.wandb_run_id = os.getenv("WANDB_RUN_ID", None)++ if self.wandb_run_id:+ # Initialize wandb with the same run ID+ wandb.init(+ project="prompt_optimizers",+ id=self.wandb_run_id,+ resume="must",+ )++ # Setup random seeds+ random.seed(seed)+ np.random.seed(seed)++ # Log current file to log_dir+ curr_file = os.path.abspath(__file__)+ save_file_to_log_dir(curr_file, self.log_dir)++ # Set up program and evaluation function+ program = student.deepcopy()+ evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)++ # Determine the number of fewshot examples to use to generate demos for prompt+ if max_bootstrapped_demos == 0 and max_labeled_demos == 0:+ max_bootstrapped_demos_for_candidate_gen = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT+ max_labeled_demos_for_candidate_gen = LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT+ else:+ max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos+ max_labeled_demos_for_candidate_gen = max_labeled_demos++ # # Generate N few shot example sets+ # if not demo_candidates and not instruction_candidates:+ # demo_candidates = create_n_fewshot_example_sets(program=program, trainset=trainset, n=self.n, hard_fewshot=hard_fewshot, metric=self.metric, teacher_settings=self.teacher_settings, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen)++ # Generate N few shot example sets+ try:+ demo_candidates = create_n_fewshot_demo_sets(+ student=program,+ num_candidate_sets=self.n,+ trainset=trainset,+ max_labeled_demos=max_labeled_demos_for_candidate_gen,+ max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,+ metric=self.metric,+ teacher_settings=self.teacher_settings,+ seed=seed,+ )+ except Exception as e:+ print(f"Error generating fewshot examples: {e}")+ print("Running without fewshot examples.")+ demo_candidates = None++ # Generate N candidate prompts+ proposer.program_aware = program_aware_proposer+ proposer.use_tip = True+ proposer.use_instruct_history = False+ proposer.set_history_randomly = False+ instruction_candidates = proposer.propose_instructions_for_program(+ trainset=trainset,+ program=program,+ demo_candidates=demo_candidates,+ N=self.n,+ prompt_model=self.prompt_model,+ T=self.init_temperature,+ trial_logs={},+ )+ for i, pred in enumerate(program.predictors()):+ instruction_candidates[i][0] = get_signature(pred).instructions++ # instruction_candidates[1][0] = "Given the question, and context, respond with the number of the document that is most relevant to answering the question in the field 'Answer' (ex. Answer: '3')."++ # Save the candidate instructions generated+ if self.log_dir:+ fp = os.path.join(self.log_dir, "instructions_to_save.pickle")+ with open(fp, "wb") as file:+ pickle.dump(instruction_candidates, file)++ # If we're doing zero-shot, reset demo_candidates to none+ if max_bootstrapped_demos == 0 and max_labeled_demos == 0:+ demo_candidates = None++ # Initialize variables to store the best program and its score+ best_score = float("-inf")+ best_program = None+ trial_logs = {}+ total_eval_calls = 0+ param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scores+ fully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of++ # Define our trial objective+ def create_objective(+ baseline_program,+ instruction_candidates,+ demo_candidates,+ evaluate,+ trainset,+ ):+ def objective(trial):+ nonlocal best_program, best_score, trial_logs, total_eval_calls # Allow access to the outer variables++ # Kick off trial+ logging.info(f"Starting trial num: {trial.number}")+ trial_logs[trial.number] = {}++ # Create a new candidate program+ candidate_program = baseline_program.deepcopy()++ # Choose set of instructions & demos to use for each predictor+ chosen_params = []+ for i, p_new in enumerate(candidate_program.predictors()):++ # Get instruction candidates / demos for our given predictor+ p_instruction_candidates = instruction_candidates[i]+ if demo_candidates:+ p_demo_candidates = demo_candidates[i]++ # Suggest the index of the instruction / demo candidate to use in our trial+ instruction_idx = trial.suggest_categorical(+ f"{i}_predictor_instruction",+ range(len(p_instruction_candidates)),+ )+ chosen_params.append(instruction_idx)+ if demo_candidates:+ demos_idx = trial.suggest_categorical(+ f"{i}_predictor_demos", range(len(p_demo_candidates)),+ )+ chosen_params.append(demo_candidates)++ # Log the selected instruction / demo candidate+ trial_logs[trial.number][+ f"{i}_predictor_instruction"+ ] = instruction_idx+ if demo_candidates:+ trial_logs[trial.number][f"{i}_predictor_demos"] = demos_idx++ logging.info(f"instruction_idx {instruction_idx}")+ if demo_candidates:+ logging.info(f"demos_idx {demos_idx}")++ # Set the instruction+ selected_instruction = p_instruction_candidates[instruction_idx]+ updated_signature = get_signature(p_new).with_instructions(+ selected_instruction,+ )+ set_signature(p_new, updated_signature)++ # Set the demos+ if demo_candidates:+ p_new.demos = p_demo_candidates[demos_idx]++ # Log assembled program+ print("CANDIDATE PROGRAM:")+ print_full_program(candidate_program)+ print("...")++ # Save the candidate program+ trial_logs[trial.number]["program_path"] = save_candidate_program(+ candidate_program, self.log_dir, trial.number,+ )++ trial_logs[trial.number]["num_eval_calls"] = 0++ # Evaluate the candidate program with relevant batch size+ batch_size = self._get_batch_size(minibatch, trainset)+ score = eval_candidate_program(+ batch_size, trainset, candidate_program, evaluate,+ )++ # Print out a full trace of the program in use+ print("FULL TRACE")+ full_trace = get_task_model_history_for_full_example(+ candidate_program, self.task_model, trainset, evaluate,+ )+ print("...")++ # Log relevant information+ print(f"Score {score}")+ categorical_key = ",".join(map(str, chosen_params))+ param_score_dict[categorical_key].append(+ (score, candidate_program),+ )+ trial_logs[trial.number]["num_eval_calls"] = batch_size+ trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)+ trial_logs[trial.number]["eval_example_call"] = full_trace+ trial_logs[trial.number]["score"] = score+ trial_logs[trial.number]["pruned"] = False+ total_eval_calls += trial_logs[trial.number]["num_eval_calls"]+ trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number]["program"] = candidate_program.deepcopy()+ if self.wandb_run_id:+ wandb.log(+ {+ "score": score,+ "num_eval_calls": trial_logs[trial.number]["num_eval_calls"],+ "total_eval_calls": total_eval_calls,+ },+ )++ # Update the best program if the current score is better, and if we're not using minibatching+ best_score_updated = False+ if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:+ print("Updating best score")+ best_score = score+ best_program = candidate_program.deepcopy()+ best_score_updated = True+++ # If we're doing minibatching, check to see if it's time to do a full eval+ if minibatch and trial.number % self.minibatch_full_eval_steps == 0:++ # Save old information as the minibatch version+ trial_logs[trial.number]["mb_score"] = score+ trial_logs[trial.number]["mb_program_path"] = trial_logs[trial.number]["program_path"]++ # Identify our best program (based on mean of scores so far, and do a full eval on it)+ highest_mean_program, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)+ full_train_score = eval_candidate_program(+ len(trainset), trainset, highest_mean_program, evaluate,+ )++ # Log relevant information+ fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}+ total_eval_calls += len(trainset)+ trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number]["full_eval"] = True+ trial_logs[trial.number]["program_path"] = save_candidate_program(+ program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number, note="full_eval",+ )+ trial_logs[trial.number]["score"] = full_train_score++ if full_train_score > best_score:+ print(f"UPDATING BEST SCORE WITH {full_train_score}")+ best_score = full_train_score+ best_program = highest_mean_program.deepcopy()+ best_score_updated = True++ # If the best score was updated, do a full eval on the dev set+ if best_score_updated:+ full_dev_score = evaluate(+ best_program,+ devset=valset,+ display_table=0,+ )+ if self.wandb_run_id:+ wandb.log(+ {+ "best_prog_so_far_train_score": best_score,+ "best_prog_so_far_dev_score": full_dev_score,+ },+ )++ return score++ return objective++ # Run the trial+ objective_function = create_objective(+ program, instruction_candidates, demo_candidates, evaluate, trainset,+ )++ # if minibatch:+ # num_batches = int(num_batches*len(trainset)/(self.minibatch_size + len(trainset)/self.minibatch_full_eval_steps))++ sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)+ study = optuna.create_study(direction="maximize", sampler=sampler)+ score = study.optimize(objective_function, n_trials=num_batches)++ if best_program is not None and self.track_stats:+ best_program.trial_logs = trial_logs+ best_program.score = best_score+ best_program.prompt_model_total_calls = self.prompt_model_total_calls+ best_program.total_calls = self.total_calls++ # program_file_path = os.path.join(self.log_dir, 'best_program.pickle')+ if self.log_dir:+ program_file_path = os.path.join(self.log_dir, "best_program")+ best_program.save(program_file_path)++ optuna_study_file_path = os.path.join(self.log_dir, "optuna_study.pickle")+ with open(optuna_study_file_path, "wb") as file:+ pickle.dump(study, file)++ return best_program+ return studentdiff --git a/examples/nli/scone/scone_with_MIPRO.ipynb b/examples/nli/scone/scone_with_MIPRO.ipynbindex 9fd3da46d..106650cc3 100644--- a/examples/nli/scone/scone_with_MIPRO.ipynb+++ b/examples/nli/scone/scone_with_MIPRO.ipynb@@ -482,7 +482,7 @@"outputs": [],"source": ["import cloudpickle as pickle\n",- "from dspy.teleprompt import MIPRO\n",+ "from dspy.teleprompt import MIPROv2\n","\n","LOAD_PRECOMPILED_PROGRAM = False\n","\n",@@ -501,7 +501,7 @@"\n"," # Compile\n"," eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",- " teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n",+ " teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n"," compiled_program = teleprompter.compile(program, trainset=trainset, valset=valset, num_batches=batches, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"]},diff --git a/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb b/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynbindex 0e45d7602..89f4c8116 100644--- a/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb@@ -455,7 +455,7 @@"outputs": [],"source": ["import cloudpickle as pickle\n",- "from dspy.teleprompt import MIPRO\n",+ "from dspy.teleprompt import MIPROv2\n","\n","LOAD_PRECOMPILED_PROGRAM = False\n","\n",@@ -474,7 +474,7 @@"\n"," # Compile\n"," eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",- " teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n",+ " teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n"," compiled_program = teleprompter.compile(program, trainset=trainset, valset=valset, num_batches=batches, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"]},diff --git a/tests/teleprompt/test_mipro_optimizer.py b/tests/teleprompt/test_mipro_optimizer.pyindex 4b0eefe14..17e94a580 100644--- a/tests/teleprompt/test_mipro_optimizer.py+++ b/tests/teleprompt/test_mipro_optimizer.py@@ -118,7 +118,7 @@ def test_bayesian_signature_optimizer_initialization():metric=simple_metric, num_candidates=10, init_temperature=1.4, verbose=True, track_stats=True)assert optimizer.metric == simple_metric, "Metric not correctly initialized"- assert optimizer.n == 10, "Incorrect 'num_candidates' parameter initialization"+ assert optimizer.num_candidates == 10, "Incorrect 'num_candidates' parameter initialization"assert (optimizer.init_temperature == 1.4), "Initial temperature not correctly initialized"@@ -154,7 +154,7 @@ def test_signature_optimizer_optimization_process():optimized_student = optimizer.compile(student=student,trainset=trainset,- num_batches=10,+ num_trials=10,max_bootstrapped_demos=3,max_labeled_demos=5,eval_kwargs={"num_threads": 1, "display_progress": False},@@ -184,7 +184,7 @@ def test_signature_optimizer_bad_lm():_optimized_student = optimizer.compile(student=student,trainset=trainset,- num_batches=10,+ num_trials=10,max_bootstrapped_demos=3,max_labeled_demos=5,eval_kwargs={"num_threads": 1, "display_progress": False},@@ -212,7 +212,7 @@ def test_optimization_and_output_verification():optimized_student = optimizer.compile(student=student,trainset=trainset,- num_batches=4,+ num_trials=4,max_bootstrapped_demos=2,max_labeled_demos=3,eval_kwargs={"num_threads": 1, "display_progress": False},