DSPy: The framework for programming—not prompting—language models
Python
MIPROv2 Updates (#8166)
klopsahlongbb5f0d1b31ce
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex d650d7da7..711b227f5 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -3,11 +3,13 @@ import randomimport textwrapfrom collections import defaultdictfrom typing import Any, Callable, Dict, List, Literal, Optional, Tuple+import select+import sys+import timeimport numpy as npimport optunafrom optuna.distributions import CategoricalDistribution-import dspyfrom dspy.evaluate.evaluate import Evaluatefrom dspy.propose import GroundedProposer@@ -53,10 +55,8 @@ class MIPROv2(Teleprompter):teacher_settings: Dict = {},max_bootstrapped_demos: int = 4,max_labeled_demos: int = 4,- auto: Optional[Literal["light", "medium", "heavy"]] = "medium",- num_candidates: int = 10,- num_fewshot_candidates: Optional[int] = None,- num_instruct_candidates: Optional[int] = None,+ auto: Optional[Literal["light", "medium", "heavy"]] = "light",+ num_candidates: Optional[int] = None,num_threads: Optional[int] = None,max_errors: int = 10,seed: int = 9,@@ -71,9 +71,9 @@ class MIPROv2(Teleprompter):if auto not in allowed_modes:raise ValueError(f"Invalid value for auto: {auto}. Must be one of {allowed_modes}.")self.auto = auto-- self.num_fewshot_candidates = num_fewshot_candidates or num_candidates- self.num_instruct_candidates = num_instruct_candidates or num_candidates+ self.num_fewshot_candidates = num_candidates+ self.num_instruct_candidates = num_candidates+ self.num_candidates = num_candidatesself.metric = metricself.init_temperature = init_temperatureself.task_model = task_model if task_model else dspy.settings.lm@@ -99,7 +99,7 @@ class MIPROv2(Teleprompter):trainset: List,teacher: Any = None,valset: Optional[List] = None,- num_trials: int = 30,+ num_trials: Optional[int] = None,max_bootstrapped_demos: Optional[int] = None,max_labeled_demos: Optional[int] = None,seed: Optional[int] = None,@@ -114,6 +114,21 @@ class MIPROv2(Teleprompter):requires_permission_to_run: bool = True,provide_traceback: Optional[bool] = None,) -> Any:++ zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)++ # If auto is None, and num_trials is not provided (but num_candidates is), raise an error that suggests a good num_trials value+ if self.auto is None and (self.num_candidates is not None and num_trials is None):+ raise ValueError(f"If auto is None, num_trials must also be provided. Given num_candidates={self.num_candidates}, we'd recommend setting num_trials to ~{self._set_num_trials_from_num_candidates(student, zeroshot_opt, self.num_candidates)}.")++ # If auto is None, and num_candidates or num_trials is None, raise an error+ if self.auto is None and (self.num_candidates is None or num_trials is None):+ raise ValueError("If auto is None, num_candidates must also be provided.")++ # If auto is provided, and either num_candidates or num_trials is not None, raise an error+ if self.auto is not None and (self.num_candidates is not None or num_trials is not None):+ raise ValueError("If auto is not None, num_candidates and num_trials cannot be set, since they would be overrided by the auto settings. Please either set auto to None, or do not specify num_candidates and num_trials.")+# Set random seedsseed = seed or self.seedself._set_random_seeds(seed)@@ -128,7 +143,6 @@ class MIPROv2(Teleprompter):trainset, valset = self._set_and_validate_datasets(trainset, valset)# Set hyperparameters based on run mode (if set)- zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(student, num_trials, minibatch, zeroshot_opt, valset)@@ -204,6 +218,15 @@ class MIPROv2(Teleprompter):self.rng = random.Random(seed)np.random.seed(seed)+ def _set_num_trials_from_num_candidates(self, program, zeroshot_opt, num_candidates):+ num_vars = len(program.predictors())+ if not zeroshot_opt:+ num_vars *= 2 # Account for few-shot examples + instruction variables+ # Trials = MAX(c*M*log(N), c=2, 3/2*N)+ num_trials = int(max(2 * num_vars * np.log2(num_candidates), 1.5 * num_candidates))++ return num_trials+def _set_hyperparams_from_run_mode(self,program: Any,@@ -226,11 +249,7 @@ class MIPROv2(Teleprompter):self.num_instruct_candidates = auto_settings["n"] if zeroshot_opt else int(auto_settings["n"] * 0.5)self.num_fewshot_candidates = auto_settings["n"]- num_vars = len(program.predictors())- if not zeroshot_opt:- num_vars *= 2 # Account for few-shot examples + instruction variables- # Trials = MAX(c*M*log(N), c=2, 3/2*N)- num_trials = max(2 * num_vars * np.log(auto_settings["n"]), 1.5 * auto_settings["n"])+ num_trials = self._set_num_trials_from_num_candidates(program, zeroshot_opt, auto_settings["n"])return num_trials, valset, minibatch@@ -353,6 +372,7 @@ class MIPROv2(Teleprompter):user_confirmation_message = textwrap.dedent(f"""\To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.+ If no input is received within 20 seconds, the program will proceed automatically.If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.@@ -360,10 +380,18 @@ class MIPROv2(Teleprompter):""")- user_input = (- input(f"{user_message}\n{user_confirmation_message}\nDo you wish to continue? (y/n): ").strip().lower()- )- return user_input == "y"+ print(f"{user_message}\n{user_confirmation_message}\nDo you wish to continue? (y/n): ", end='', flush=True)++ # Wait for input with timeout+ start_time = time.time()+ while time.time() - start_time < 20:+ if select.select([sys.stdin], [], [], 0.1)[0]:+ user_input = sys.stdin.readline().strip().lower()+ return user_input == "y"+ time.sleep(0.1)++ print("\nNo input received within 20 seconds. Proceeding with execution...")+ return Truedef _bootstrap_fewshot_examples(self, program: Any, trainset: List, seed: int, teacher: Any) -> Optional[List]:logger.info("\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")