Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

MIPROv2 Updates (#8166)

klopsahlongbb5f0d1b31ce
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index d650d7da7..711b227f5 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -3,11 +3,13 @@ import random
import textwrap
from collections import defaultdict
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
+import select
+import sys
+import time
import numpy as np
import optuna
from optuna.distributions import CategoricalDistribution
-
import dspy
from dspy.evaluate.evaluate import Evaluate
from dspy.propose import GroundedProposer
@@ -53,10 +55,8 @@ class MIPROv2(Teleprompter):
teacher_settings: Dict = {},
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 4,
- auto: Optional[Literal["light", "medium", "heavy"]] = "medium",
- num_candidates: int = 10,
- num_fewshot_candidates: Optional[int] = None,
- num_instruct_candidates: Optional[int] = None,
+ auto: Optional[Literal["light", "medium", "heavy"]] = "light",
+ num_candidates: Optional[int] = None,
num_threads: Optional[int] = None,
max_errors: int = 10,
seed: int = 9,
@@ -71,9 +71,9 @@ class MIPROv2(Teleprompter):
if auto not in allowed_modes:
raise ValueError(f"Invalid value for auto: {auto}. Must be one of {allowed_modes}.")
self.auto = auto
-
- self.num_fewshot_candidates = num_fewshot_candidates or num_candidates
- self.num_instruct_candidates = num_instruct_candidates or num_candidates
+ self.num_fewshot_candidates = num_candidates
+ self.num_instruct_candidates = num_candidates
+ self.num_candidates = num_candidates
self.metric = metric
self.init_temperature = init_temperature
self.task_model = task_model if task_model else dspy.settings.lm
@@ -99,7 +99,7 @@ class MIPROv2(Teleprompter):
trainset: List,
teacher: Any = None,
valset: Optional[List] = None,
- num_trials: int = 30,
+ num_trials: Optional[int] = None,
max_bootstrapped_demos: Optional[int] = None,
max_labeled_demos: Optional[int] = None,
seed: Optional[int] = None,
@@ -114,6 +114,21 @@ class MIPROv2(Teleprompter):
requires_permission_to_run: bool = True,
provide_traceback: Optional[bool] = None,
) -> Any:
+
+ zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)
+
+ # If auto is None, and num_trials is not provided (but num_candidates is), raise an error that suggests a good num_trials value
+ if self.auto is None and (self.num_candidates is not None and num_trials is None):
+ raise ValueError(f"If auto is None, num_trials must also be provided. Given num_candidates={self.num_candidates}, we'd recommend setting num_trials to ~{self._set_num_trials_from_num_candidates(student, zeroshot_opt, self.num_candidates)}.")
+
+ # If auto is None, and num_candidates or num_trials is None, raise an error
+ if self.auto is None and (self.num_candidates is None or num_trials is None):
+ raise ValueError("If auto is None, num_candidates must also be provided.")
+
+ # If auto is provided, and either num_candidates or num_trials is not None, raise an error
+ if self.auto is not None and (self.num_candidates is not None or num_trials is not None):
+ raise ValueError("If auto is not None, num_candidates and num_trials cannot be set, since they would be overrided by the auto settings. Please either set auto to None, or do not specify num_candidates and num_trials.")
+
# Set random seeds
seed = seed or self.seed
self._set_random_seeds(seed)
@@ -128,7 +143,6 @@ class MIPROv2(Teleprompter):
trainset, valset = self._set_and_validate_datasets(trainset, valset)
# Set hyperparameters based on run mode (if set)
- zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)
num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(
student, num_trials, minibatch, zeroshot_opt, valset
)
@@ -204,6 +218,15 @@ class MIPROv2(Teleprompter):
self.rng = random.Random(seed)
np.random.seed(seed)
+ def _set_num_trials_from_num_candidates(self, program, zeroshot_opt, num_candidates):
+ num_vars = len(program.predictors())
+ if not zeroshot_opt:
+ num_vars *= 2 # Account for few-shot examples + instruction variables
+ # Trials = MAX(c*M*log(N), c=2, 3/2*N)
+ num_trials = int(max(2 * num_vars * np.log2(num_candidates), 1.5 * num_candidates))
+
+ return num_trials
+
def _set_hyperparams_from_run_mode(
self,
program: Any,
@@ -226,11 +249,7 @@ class MIPROv2(Teleprompter):
self.num_instruct_candidates = auto_settings["n"] if zeroshot_opt else int(auto_settings["n"] * 0.5)
self.num_fewshot_candidates = auto_settings["n"]
- num_vars = len(program.predictors())
- if not zeroshot_opt:
- num_vars *= 2 # Account for few-shot examples + instruction variables
- # Trials = MAX(c*M*log(N), c=2, 3/2*N)
- num_trials = max(2 * num_vars * np.log(auto_settings["n"]), 1.5 * auto_settings["n"])
+ num_trials = self._set_num_trials_from_num_candidates(program, zeroshot_opt, auto_settings["n"])
return num_trials, valset, minibatch
@@ -353,6 +372,7 @@ class MIPROv2(Teleprompter):
user_confirmation_message = textwrap.dedent(
f"""\
To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
+ If no input is received within 20 seconds, the program will proceed automatically.
If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.
@@ -360,10 +380,18 @@ class MIPROv2(Teleprompter):
"""
)
- user_input = (
- input(f"{user_message}\n{user_confirmation_message}\nDo you wish to continue? (y/n): ").strip().lower()
- )
- return user_input == "y"
+ print(f"{user_message}\n{user_confirmation_message}\nDo you wish to continue? (y/n): ", end='', flush=True)
+
+ # Wait for input with timeout
+ start_time = time.time()
+ while time.time() - start_time < 20:
+ if select.select([sys.stdin], [], [], 0.1)[0]:
+ user_input = sys.stdin.readline().strip().lower()
+ return user_input == "y"
+ time.sleep(0.1)
+
+ print("\nNo input received within 20 seconds. Proceeding with execution...")
+ return True
def _bootstrap_fewshot_examples(self, program: Any, trainset: List, seed: int, teacher: Any) -> Optional[List]:
logger.info("\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")