Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

adding auto run mode for miprov2

klopsahlongbe581df924dc
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.py
index d1d6192bb..740197f6f 100644
--- a/dspy/propose/grounded_proposer.py
+++ b/dspy/propose/grounded_proposer.py
@@ -270,20 +270,27 @@ class GroundedProposer(Proposer):
self.verbose = verbose
self.prompt_model = get_prompt_model(prompt_model)
+
+ self.program_code_string = None
if self.program_aware:
try:
self.program_code_string = get_dspy_source_code(program)
if self.verbose: print("SOURCE CODE:",self.program_code_string)
except Exception as e:
print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")
- self.program_code_string = None
self.program_aware = False
- else:
- self.program_code_string = None
- self.data_summary = create_dataset_summary(
- trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,
- )
- if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")
+
+ self.data_summary = None
+ if self.use_dataset_summary:
+ try:
+ self.data_summary = create_dataset_summary(
+ trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,
+ )
+ if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")
+ except Exception as e:
+ print(f"Error getting data summary: {e}.\n\nRunning without data aware proposer.")
+ self.use_dataset_summary = False
+ print(f"")
def propose_instructions_for_program(
self,
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index ab758f7e6..6e238e383 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -19,6 +19,7 @@ from dspy.teleprompt.utils import (
print_full_program,
save_candidate_program,
set_signature,
+ create_minibatch,
)
"""
@@ -63,13 +64,45 @@ Note that this teleprompter takes in the following parameters:
BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3
LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0
+MIN_MINIBATCH_SIZE = 50
+
+auto_run_settings = {
+ "light": {"num_trials": 7, "val_size": 100},
+ "medium": {"num_trials": 25, "val_size": 300},
+ "heavy": {"num_trials": 50, "val_size": 1000},
+}
+
+
class MIPROv2(Teleprompter):
+
+ def _set_hyperparams_from_run_mode(
+ self, program, num_trials, minibatch, zeroshot_opt, valset
+ ):
+ if self.auto_run_mode is None:
+ return num_trials, valset, minibatch
+
+ num_vars = (
+ len(program.predictors()) if zeroshot_opt else len(program.predictors()) * 2
+ ) # (multiplying by two to account for few-shot examples + instruction variables)
+
+ num_trials = auto_run_settings[self.auto_run_mode]["num_trials"]
+ valset = create_minibatch(
+ valset, batch_size=auto_run_settings[self.auto_run_mode]["val_size"]
+ )
+ minibatch = len(valset) > MIN_MINIBATCH_SIZE
+ self.num_candidates = int(
+ np.round(np.min([num_trials * num_vars, (3 / 2 * num_trials) / num_vars]))
+ )
+
+ return num_trials, valset, minibatch
+
def __init__(
self,
metric,
prompt_model=dspy.settings.lm,
task_model=dspy.settings.lm,
teacher_settings={},
+ auto_run_mode=None,
num_candidates=10,
num_threads=6,
max_errors=10,
@@ -79,6 +112,14 @@ class MIPROv2(Teleprompter):
log_dir=None,
metric_threshold=None,
):
+ # Check for valid auto_run_mode
+ allowed_modes = {None, "light", "medium", "heavy"}
+ if auto_run_mode not in allowed_modes:
+ raise ValueError(
+ f"Invalid value for auto_run_mode: {auto_run_mode}. Must be one of {allowed_modes}."
+ )
+ self.auto_run_mode = auto_run_mode
+
self.num_candidates = num_candidates
self.metric = metric
self.init_temperature = init_temperature
@@ -129,27 +170,48 @@ class MIPROv2(Teleprompter):
if not valset:
if len(trainset) < 2:
- raise ValueError("Trainset must have at least 2 examples if no valset specified, or at least 1 example with external validation set.")
-
- valset_size = min(500, max(1, int(len(trainset) * 0.80))) # 80% of trainset, capped at 500
+ raise ValueError(
+ "Trainset must have at least 2 examples if no valset specified, or at least 1 example with external validation set."
+ )
+
+ valset_size = min(
+ 500, max(1, int(len(trainset) * 0.80))
+ ) # 80% of trainset, capped at 500
cutoff = len(trainset) - valset_size
valset = trainset[cutoff:]
trainset = trainset[:cutoff]
else:
if len(valset) < 1:
- raise ValueError("Validation set must have at least 1 example if specified.")
-
+ raise ValueError(
+ "Validation set must have at least 1 example if specified."
+ )
+
+ # Set hyperparams
+ zeroshot_opt = (max_bootstrapped_demos == 0) and (max_labeled_demos == 0)
+ num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(
+ student, num_trials, minibatch, zeroshot_opt, valset
+ )
+
+ if self.auto_run_mode:
+ print(
+ f"RUNNING WITH THE FOLLOWING {self.auto_run_mode.upper()} AUTO RUN SETTINGS:\nnum_trials: {num_trials}\nminibatch: {minibatch}\nnum_candidates: {self.num_candidates}\nvalset size: {len(valset)}P"
+ )
+
if minibatch and minibatch_size > len(valset):
- raise ValueError(f"Minibatch size cannot exceed the size of the valset. Note that your validation set contains {len(valset)} examples. Your train set contains {len(trainset)} examples.")
-
- if minibatch and num_trials < minibatch_full_eval_steps:
- raise ValueError(f"Number of trials (num_trials={num_trials}) must be greater than or equal to the number of minibatch full eval steps (minibatch_full_eval_steps={minibatch_full_eval_steps}).")
+ raise ValueError(
+ f"Minibatch size cannot exceed the size of the valset. Note that your validation set contains {len(valset)} examples. Your train set contains {len(trainset)} examples."
+ )
- estimated_prompt_model_calls = 10 + self.num_candidates * len(
+ estimated_prompt_model_calls = (
+ 10
+ + self.num_candidates
+ * len(
student.predictors(),
- ) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)
-
+ )
+ + (0 if not program_aware_proposer else len(student.predictors()) + 1)
+ ) # num data summary calls + N * P + (P + 1)
+
prompt_model_line = ""
if not program_aware_proposer:
prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""
@@ -159,14 +221,18 @@ class MIPROv2(Teleprompter):
estimated_task_model_calls_wo_module_calls = 0
task_model_line = ""
if not minibatch:
- estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P
+ estimated_task_model_calls_wo_module_calls = (
+ len(trainset) * num_trials
+ ) # M * T * P
task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(valset)}{ENDC}{YELLOW} examples in val set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""
else:
- estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (len(trainset) * (num_trials // minibatch_full_eval_steps)) # B * T * P
+ estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (
+ len(trainset) * (num_trials // minibatch_full_eval_steps)
+ ) # B * T * P
task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(valset)}{ENDC}{YELLOW} examples in val set * {BLUE}{BOLD}{num_trials // minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""
-
- user_message = textwrap.dedent(f"""\
+ user_message = textwrap.dedent(
+ f"""\
{YELLOW}{BOLD}Projected Language Model (LM) Calls{ENDC}
Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:
@@ -184,17 +250,21 @@ class MIPROv2(Teleprompter):
and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:
{YELLOW}- Reducing the number of trials (`num_trials`), the size of the valset, or the number of LM calls in your program.{ENDC}
- {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n""")
+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n"""
+ )
- user_confirmation_message = textwrap.dedent(f"""\
+ user_confirmation_message = textwrap.dedent(
+ f"""\
To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.
{YELLOW}Awaiting your input...{ENDC}
- """)
+ """
+ )
- if requires_permission_to_run: print(user_message)
+ if requires_permission_to_run:
+ print(user_message)
sys.stdout.flush() # Flush the output buffer to force the message to print
@@ -225,17 +295,23 @@ class MIPROv2(Teleprompter):
# Determine the number of fewshot examples to use to generate demos for prompt
if max_bootstrapped_demos == 0 and max_labeled_demos == 0:
- max_bootstrapped_demos_for_candidate_gen = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT
- max_labeled_demos_for_candidate_gen = LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT
+ max_bootstrapped_demos_for_candidate_gen = (
+ BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT
+ )
+ max_labeled_demos_for_candidate_gen = (
+ LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT
+ )
else:
max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos
max_labeled_demos_for_candidate_gen = max_labeled_demos
-
+
# Generate N few shot example sets (these will inform instruction creation, and be used as few-shot examples in our prompt)
print("Beginning MIPROv2 optimization process...")
print("\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")
if max_bootstrapped_demos > 0:
- print("These will be used for as few-shot examples candidates for our program and for creating instructions.\n")
+ print(
+ "These will be used for as few-shot examples candidates for our program and for creating instructions.\n"
+ )
else:
print("These will be used for informing instruction proposal.\n")
print(f"Bootstrapping N={self.num_candidates} sets of demonstrations...")
@@ -256,12 +332,14 @@ class MIPROv2(Teleprompter):
print(f"Error generating fewshot examples: {e}")
print("Running without fewshot examples.")
demo_candidates = None
-
+
# Generate N candidate instructions
- # Setup our proposer
+ # Setup our proposer
print("\n==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==")
- print("In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.")
+ print(
+ "In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions."
+ )
proposer = GroundedProposer(
program=program,
@@ -276,9 +354,9 @@ class MIPROv2(Teleprompter):
set_tip_randomly=tip_aware_proposer,
use_instruct_history=False,
set_history_randomly=False,
- verbose = self.verbose,
+ verbose=self.verbose,
)
-
+
print("\nProposing instructions...\n")
instruction_candidates = proposer.propose_instructions_for_program(
trainset=trainset,
@@ -303,18 +381,23 @@ class MIPROv2(Teleprompter):
best_scoring_trial = 0
trial_logs = {}
total_eval_calls = 0
- param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scores
- fully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of
+ param_score_dict = defaultdict(
+ list
+ ) # Dictionaries of paramater combinations we've tried, and their associated scores
+ fully_evaled_param_combos = (
+ {}
+ ) # List of the parameter combinations we've done full evals of
# Evaluate the default program
print("Evaluating the default program...\n")
- default_score = eval_candidate_program(len(valset), valset, program, evaluate)
+ default_score = eval_candidate_program(
+ len(valset), valset, program, evaluate
+ )
print(f"Default program score: {default_score}\n")
best_score = default_score
best_program = program.deepcopy()
-
# Define our trial objective
def create_objective(
baseline_program,
@@ -325,13 +408,13 @@ class MIPROv2(Teleprompter):
):
def objective(trial):
nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls # Allow access to the outer variables
-
+
# Kick off trial
if minibatch:
print(f"== Minibatch Trial {trial.number+1} / {num_trials} ==")
else:
print(f"===== Trial {trial.number+1} / {num_trials} =====")
- trial_logs[trial.number+1] = {}
+ trial_logs[trial.number + 1] = {}
# Create a new candidate program
candidate_program = baseline_program.deepcopy()
@@ -351,19 +434,26 @@ class MIPROv2(Teleprompter):
range(len(p_instruction_candidates)),
)
# chosen_params.append(instruction_idx)
- chosen_params.append(f"Predictor {i+1}: Instruction {instruction_idx}")
+ chosen_params.append(
+ f"Predictor {i+1}: Instruction {instruction_idx}"
+ )
if demo_candidates:
demos_idx = trial.suggest_categorical(
- f"{i}_predictor_demos", range(len(p_demo_candidates)),
+ f"{i}_predictor_demos",
+ range(len(p_demo_candidates)),
+ )
+ chosen_params.append(
+ f"Predictor {i+1}: Few-Shot Set {demos_idx}"
)
- chosen_params.append(f"Predictor {i+1}: Few-Shot Set {demos_idx}")
# Log the selected instruction / demo candidate
- trial_logs[trial.number+1][
+ trial_logs[trial.number + 1][
f"{i}_predictor_instruction"
] = instruction_idx
if demo_candidates:
- trial_logs[trial.number+1][f"{i}_predictor_demos"] = demos_idx
+ trial_logs[trial.number + 1][
+ f"{i}_predictor_demos"
+ ] = demos_idx
dspy.logger.debug(f"instruction_idx {instruction_idx}")
if demo_candidates:
@@ -381,21 +471,29 @@ class MIPROv2(Teleprompter):
p_new.demos = p_demo_candidates[demos_idx]
# Log assembled program
- if self.verbose: print("Evaluating the following candidate program...\n")
- if self.verbose: print_full_program(candidate_program)
+ if self.verbose:
+ print("Evaluating the following candidate program...\n")
+ if self.verbose:
+ print_full_program(candidate_program)
# Save the candidate program
- trial_logs[trial.number+1]["program_path"] = save_candidate_program(
- candidate_program, self.log_dir, trial.number+1,
+ trial_logs[trial.number + 1]["program_path"] = (
+ save_candidate_program(
+ candidate_program,
+ self.log_dir,
+ trial.number + 1,
+ )
)
- trial_logs[trial.number+1]["num_eval_calls"] = 0
+ trial_logs[trial.number + 1]["num_eval_calls"] = 0
# Evaluate the candidate program with relevant batch size
batch_size = minibatch_size if minibatch else len(valset)
score = eval_candidate_program(
- batch_size, valset, candidate_program,
+ batch_size,
+ valset,
+ candidate_program,
evaluate,
)
@@ -403,7 +501,10 @@ class MIPROv2(Teleprompter):
if self.verbose:
print("Full trace of prompts in use on an example...")
get_task_model_history_for_full_example(
- candidate_program, self.task_model, valset, evaluate,
+ candidate_program,
+ self.task_model,
+ valset,
+ evaluate,
)
# Log relevant information
@@ -411,72 +512,117 @@ class MIPROv2(Teleprompter):
param_score_dict[categorical_key].append(
(score, candidate_program),
)
- trial_logs[trial.number+1]["num_eval_calls"] = batch_size
- trial_logs[trial.number+1]["full_eval"] = batch_size >= len(valset)
- trial_logs[trial.number+1]["score"] = score
- trial_logs[trial.number+1]["pruned"] = False
- total_eval_calls += trial_logs[trial.number+1]["num_eval_calls"]
- trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls
- trial_logs[trial.number+1]["program"] = candidate_program.deepcopy()
+ trial_logs[trial.number + 1]["num_eval_calls"] = batch_size
+ trial_logs[trial.number + 1]["full_eval"] = batch_size >= len(
+ valset
+ )
+ trial_logs[trial.number + 1]["score"] = score
+ trial_logs[trial.number + 1]["pruned"] = False
+ total_eval_calls += trial_logs[trial.number + 1]["num_eval_calls"]
+ trial_logs[trial.number + 1][
+ "total_eval_calls_so_far"
+ ] = total_eval_calls
+ trial_logs[trial.number + 1][
+ "program"
+ ] = candidate_program.deepcopy()
# If this score was from a full evaluation, update the best program if the new score is better
best_score_updated = False
- if score > best_score and trial_logs[trial.number+1]["full_eval"] and not minibatch:
+ if (
+ score > best_score
+ and trial_logs[trial.number + 1]["full_eval"]
+ and not minibatch
+ ):
best_score = score
- best_scoring_trial = trial.number+1
+ best_scoring_trial = trial.number + 1
best_program = candidate_program.deepcopy()
best_score_updated = True
-
+
if minibatch:
- print(f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n")
+ print(
+ f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n"
+ )
else:
print(f"Score: {score} with parameters {chosen_params}.")
if best_score_updated:
- print(f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n")
+ print(
+ f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n"
+ )
else:
- print(f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n")
-
+ print(
+ f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n"
+ )
# If we're doing minibatching, check to see if it's time to do a full eval
- if minibatch and (((trial.number+1) % minibatch_full_eval_steps == 0) or (trial.number+1 == num_trials)):
- print(f"===== Full Eval {len(fully_evaled_param_combos)+1} =====")
-
+ if minibatch and (
+ ((trial.number + 1) % minibatch_full_eval_steps == 0)
+ or (trial.number + 1 == num_trials)
+ ):
+ print(
+ f"===== Full Eval {len(fully_evaled_param_combos)+1} ====="
+ )
+
# Save old information as the minibatch version
- trial_logs[trial.number+1]["mb_score"] = score
- trial_logs[trial.number+1]["mb_program_path"] = trial_logs[trial.number+1]["program_path"]
+ trial_logs[trial.number + 1]["mb_score"] = score
+ trial_logs[trial.number + 1]["mb_program_path"] = trial_logs[
+ trial.number + 1
+ ]["program_path"]
# Identify our best program (based on mean of scores so far, and do a full eval on it)
- highest_mean_program, mean, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)
-
- if trial.number+1 // minibatch_full_eval_steps > 0:
- print(f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials...")
+ highest_mean_program, mean, combo_key = (
+ get_program_with_highest_avg_score(
+ param_score_dict, fully_evaled_param_combos
+ )
+ )
+
+ if trial.number + 1 // minibatch_full_eval_steps > 0:
+ print(
+ f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials..."
+ )
else:
- print(f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials...")
+ print(
+ f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials..."
+ )
full_val_score = eval_candidate_program(
- len(valset), valset, highest_mean_program, evaluate,
+ len(valset),
+ valset,
+ highest_mean_program,
+ evaluate,
)
# Log relevant information
- fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_val_score}
+ fully_evaled_param_combos[combo_key] = {
+ "program": highest_mean_program,
+ "score": full_val_score,
+ }
total_eval_calls += len(valset)
- trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls
- trial_logs[trial.number+1]["full_eval"] = True
- trial_logs[trial.number+1]["program_path"] = save_candidate_program(
- program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number+1, note="full_eval",
+ trial_logs[trial.number + 1][
+ "total_eval_calls_so_far"
+ ] = total_eval_calls
+ trial_logs[trial.number + 1]["full_eval"] = True
+ trial_logs[trial.number + 1]["program_path"] = (
+ save_candidate_program(
+ program=highest_mean_program,
+ log_dir=self.log_dir,
+ trial_num=trial.number + 1,
+ note="full_eval",
+ )
)
- trial_logs[trial.number+1]["score"] = full_val_score
-
+ trial_logs[trial.number + 1]["score"] = full_val_score
+
if full_val_score > best_score:
- print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_val_score}")
+ print(
+ f"{GREEN}Best full eval score so far!{ENDC} Score: {full_val_score}"
+ )
best_score = full_val_score
- best_scoring_trial = trial.number+1
+ best_scoring_trial = trial.number + 1
best_program = highest_mean_program.deepcopy()
best_score_updated = True
else:
print(f"Full eval score: {full_val_score}")
print(f"Best full eval score so far: {best_score}")
print("=======================\n\n")
-
+
return score
return objective
@@ -484,9 +630,15 @@ class MIPROv2(Teleprompter):
# Run the trial
optuna.logging.set_verbosity(optuna.logging.WARNING)
print("==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==")
- print("In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n")
+ print(
+ "In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n"
+ )
objective_function = create_objective(
- program, instruction_candidates, demo_candidates, evaluate, valset,
+ program,
+ instruction_candidates,
+ demo_candidates,
+ evaluate,
+ valset,
)
sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)
study = optuna.create_study(direction="maximize", sampler=sampler)