Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

remove commented out optimizers (#7971)

TomuHirata0db6898d0345
diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.py
deleted file mode 100644
index 8a9899c4e..000000000
--- a/dspy/teleprompt/mipro_optimizer.py
+++ /dev/null
@@ -1,599 +0,0 @@
-# import math
-# import random
-# import sys
-# import textwrap
-# from collections import defaultdict
-# from typing import Any
-
-# import optuna
-
-# import dspy
-# from dspy.evaluate.evaluate import Evaluate
-# from dspy.signatures import Signature
-# from dspy.signatures.signature import signature_to_template
-# from dspy.teleprompt import BootstrapFewShot
-# from dspy.teleprompt.teleprompt import Teleprompter
-
-# """
-# USAGE SUGGESTIONS:
-
-# The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:
-
-# ``` python
-# from dspy.teleprompt import MIPRO
-
-# teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)
-# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
-# compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
-# eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
-# ```
-
-# Note that this teleprompter takes in the following parameters:
-
-# * prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).
-# * task_model: The model used for running your task. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).
-# * metric: The task metric used for optimization.
-# * num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.
-# * init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.
-# * verbose: Tells the method whether or not to print intermediate steps.
-# * track_stats: Tells the method whether or not to track statistics about the optimization process.
-# If True, the method will track a dictionary with a key corresponding to the trial number,
-# and a value containing a dict with the following keys:
-# * program: the program being evaluated at a given trial
-# * score: the last average evaluated score for the program
-# * pruned: whether or not this program was pruned
-# This information will be returned as attributes of the best program.
-# """
-
-
-# class BasicGenerateInstruction(Signature):
-# """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
-# basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
-# proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
-# proposed_prefix_for_output_field = dspy.OutputField(
-# desc="The string at the end of the prompt, which will help the model start solving the task",
-# )
-
-
-# class BasicGenerateInstructionWithDataObservations(Signature):
-# """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
-# basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
-# observations = dspy.InputField(desc="Observations about the dataset and task")
-# proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
-# proposed_prefix_for_output_field = dspy.OutputField(
-# desc="The string at the end of the prompt, which will help the model start solving the task",
-# )
-
-
-# class BasicGenerateInstructionWithExamples(dspy.Signature):
-# """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.
-
-# Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
-# # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")
-# basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
-# # examples = dspy.InputField(desc="Example(s) of the task")
-# examples = dspy.InputField(desc="Example(s) of the task")
-# proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
-# proposed_prefix_for_output_field = dspy.OutputField(
-# desc="The string at the end of the prompt, which will help the model start solving the task",
-# )
-
-
-# class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):
-# """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.
-
-# Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
-# observations = dspy.InputField(desc="Observations about the dataset and task")
-# examples = dspy.InputField(desc="Example(s) of the task")
-# basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
-# proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
-# proposed_prefix_for_output_field = dspy.OutputField(
-# desc="The string at the end of the prompt, which will help the model start solving the task",
-# )
-
-
-# class ObservationSummarizer(dspy.Signature):
-# """Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."""
-
-# observations = dspy.InputField(desc="Observations I have made about my dataset")
-# summary = dspy.OutputField(
-# desc="Two to Three sentence summary of only the most significant highlights of my observations",
-# )
-
-
-# class DatasetDescriptor(dspy.Signature):
-# (
-# """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
-# """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
-# """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""
-# )
-
-# examples = dspy.InputField(desc="Sample data points from the dataset")
-# observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")
-
-
-# class DatasetDescriptorWithPriorObservations(dspy.Signature):
-# (
-# """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
-# """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """
-# """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
-# """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""
-# )
-
-# examples = dspy.InputField(desc="Sample data points from the dataset")
-# prior_observations = dspy.InputField(desc="Some prior observations I made about the data")
-# observations = dspy.OutputField(
-# desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add",
-# )
-
-
-# class MIPRO(Teleprompter):
-# def __init__(
-# self,
-# metric,
-# prompt_model=None,
-# task_model=None,
-# teacher_settings={},
-# num_candidates=10,
-# init_temperature=1.0,
-# verbose=False,
-# track_stats=True,
-# view_data_batch_size=10,
-# ):
-# self.num_candidates = num_candidates
-# self.metric = metric
-# self.init_temperature = init_temperature
-# self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm
-# self.task_model = task_model if task_model is not None else dspy.settings.lm
-# self.verbose = verbose
-# self.track_stats = track_stats
-# self.teacher_settings = teacher_settings
-# self.view_data_batch_size = view_data_batch_size
-
-# def _print_full_program(self, program):
-# for i, predictor in enumerate(program.predictors()):
-# if self.verbose:
-# print(f"Predictor {i}")
-# if self.verbose:
-# print(f"i: {self._get_signature(predictor).instructions}")
-# *_, last_field = self._get_signature(predictor).fields.values()
-# if self.verbose:
-# print(f"p: {last_field.json_schema_extra['prefix']}")
-# if self.verbose:
-# print("\n")
-
-# def _print_model_history(self, model, n=1):
-# if self.verbose:
-# print(f"Model ({model}) History:")
-# model.inspect_history(n=n)
-
-# def _observe_data(self, trainset, max_iterations=10):
-# upper_lim = min(len(trainset), self.view_data_batch_size)
-# observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))
-# observations = observation["observations"]
-
-# skips = 0
-# iterations = 0
-# for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):
-# upper_lim = min(len(trainset), b + self.view_data_batch_size)
-# output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(
-# prior_observations=observations,
-# examples=(trainset[b:upper_lim].__repr__()),
-# )
-# iterations += 1
-# if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":
-# skips += 1
-# if skips >= 5:
-# break
-# continue
-# if iterations >= max_iterations:
-# break
-# observations += output["observations"]
-
-# summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
-
-# return summary.summary
-
-# def _create_example_string(self, fields, example):
-# # Building the output string
-# output = []
-# for field in fields:
-# name = field.name
-# separator = field.separator
-# input_variable = field.input_variable
-
-# # Determine the value from input_data or prediction_data
-# value = example.get(input_variable)
-
-# # Construct the string for the current field
-# field_str = f"{name}{separator}{value}"
-# output.append(field_str)
-
-# # Joining all the field strings
-# return "\n".join(output)
-
-# def _get_signature(self, predictor):
-# if hasattr(predictor, "extended_signature"):
-# return predictor.extended_signature
-# elif hasattr(predictor, "signature"):
-# return predictor.signature
-# return None
-
-# def _set_signature(self, predictor, updated_signature):
-# if hasattr(predictor, "extended_signature"):
-# predictor.extended_signature = updated_signature
-# elif hasattr(predictor, "signature"):
-# predictor.signature = updated_signature
-
-# def _generate_first_N_candidates( # noqa: N802
-# self,
-# module: dspy.Module,
-# N: int, # noqa: N803
-# view_data: bool,
-# view_examples: bool,
-# demo_candidates: dict,
-# devset,
-# ) -> tuple[dict, dict]:
-# candidates = {}
-# evaluated_candidates = defaultdict(dict)
-
-# if view_data:
-# # Create data observations
-# self.observations = None
-# with dspy.settings.context(lm=self.prompt_model):
-# self.observations = self._observe_data(devset).replace("Observations:", "").replace("Summary:", "")
-
-# if view_examples:
-# example_sets = {}
-# for predictor in module.predictors():
-# # Get all augmented examples
-# example_set = {}
-# all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples
-# for example_set_i, set_of_examples in enumerate(all_sets_of_examples):
-# if example_set_i != 0: # Skip the no examples case
-# for example in set_of_examples: # Get each individual example in the set
-# if "augmented" in example and example["augmented"]:
-# if example_set_i not in example_set:
-# example_set[example_set_i] = []
-# fields_to_use = signature_to_template(predictor.signature).fields
-# _input_variable_names = list(self._get_signature(predictor).input_fields.keys())
-# example_string = self._create_example_string(fields_to_use, example)
-# example_set[example_set_i].append(example_string)
-# example_sets[id(predictor)] = example_set
-# else:
-# example_set[example_set_i] = []
-# example_sets[id(predictor)] = example_set
-
-# # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
-# for predictor in module.predictors():
-# basic_instruction = None
-# basic_prefix = None
-# basic_instruction = self._get_signature(predictor).instructions
-# *_, last_field = self._get_signature(predictor).fields.values()
-# basic_prefix = last_field.json_schema_extra["prefix"]
-# with dspy.settings.context(lm=self.prompt_model):
-# # Data & Examples
-# if view_data and view_examples:
-# if 1 not in example_sets[id(predictor)]:
-# raise ValueError("No examples found for the given predictor")
-# instruct = None
-# for i in range(1, self.num_candidates):
-# new_instruct = dspy.Predict(
-# BasicGenerateInstructionWithExamplesAndDataObservations,
-# n=1,
-# temperature=self.init_temperature,
-# )(
-# basic_instruction=basic_instruction,
-# observations=self.observations,
-# examples=example_sets[id(predictor)][i],
-# )
-# if not instruct:
-# instruct = new_instruct
-# else:
-# instruct.completions.proposed_instruction.extend(
-# new_instruct.completions.proposed_instruction,
-# )
-# instruct.completions.proposed_prefix_for_output_field.extend(
-# new_instruct.completions.proposed_prefix_for_output_field,
-# )
-# # Just data
-# elif view_data:
-# instruct = dspy.Predict(
-# BasicGenerateInstructionWithDataObservations,
-# n=N - 1,
-# temperature=self.init_temperature,
-# )(basic_instruction=basic_instruction, observations=self.observations)
-# # Just examples
-# elif view_examples:
-# instruct = None
-# for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty
-# new_instruct = dspy.Predict(
-# BasicGenerateInstructionWithExamples,
-# n=1,
-# temperature=self.init_temperature,
-# )(
-# basic_instruction=basic_instruction,
-# examples=example_sets[id(predictor)][i],
-# )
-# if not instruct:
-# instruct = new_instruct
-# else:
-# instruct.completions.proposed_instruction.extend(
-# new_instruct.completions.proposed_instruction,
-# )
-# instruct.completions.proposed_prefix_for_output_field.extend(
-# new_instruct.completions.proposed_prefix_for_output_field,
-# )
-# # Neither
-# else:
-# instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)(
-# basic_instruction=basic_instruction,
-# )
-
-# # Add in our initial prompt as a candidate as well
-# instruct.completions.proposed_instruction.insert(0, basic_instruction)
-# instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)
-# candidates[id(predictor)] = instruct.completions
-# evaluated_candidates[id(predictor)] = {}
-
-# if self.verbose:
-# self._print_model_history(self.prompt_model)
-
-# return candidates, evaluated_candidates
-
-# def compile(
-# self,
-# student: dspy.Program,
-# *,
-# trainset: list[dspy.Example],
-# num_trials: int,
-# max_bootstrapped_demos: int,
-# max_labeled_demos: int,
-# eval_kwargs: dict[str, Any],
-# seed=42,
-# view_data=True,
-# view_examples=True,
-# requires_permission_to_run=True,
-# ) -> dspy.Program:
-# # Define ANSI escape codes for colors
-# RED = "\033[91m"
-# YELLOW = "\033[93m"
-# BLUE = "\033[94m"
-# BOLD = "\033[1m"
-# ENDC = "\033[0m" # Resets the color to default
-
-# random.seed(seed)
-
-# estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P
-# estimated_prompt_model_calls = 10 + self.num_candidates * len(
-# student.predictors(),
-# ) # num data summary calls + N * P
-
-# user_message = textwrap.dedent(f"""\
-# {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}
-
-# Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:
-
-# {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}
-# {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}
-
-# {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}
-
-# {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token)
-# + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC}
-
-# For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task
-# and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:
-
-# {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}
-# {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")
-
-# user_confirmation_message = textwrap.dedent(f"""\
-# To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
-
-# If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}
-
-# {YELLOW}Awaiting your input...{ENDC}
-# """)
-
-# print(f"""{RED}{BOLD}WARNING: MIPRO has been deprecated and replaced with MIPROv2. MIPRO will be removed in a future release. {ENDC}""")
-# print(user_message)
-
-# sys.stdout.flush() # Flush the output buffer to force the message to print
-
-# run = True
-# if requires_permission_to_run:
-# print(user_confirmation_message)
-# user_input = input("Do you wish to continue? (y/n): ").strip().lower()
-# if user_input != "y":
-# print("Compilation aborted by the user.")
-# run = False
-
-# if run:
-# # Set up program and evaluation function
-# module = student.deepcopy()
-# evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)
-
-# # In the case where the bootstrapped and labeled demos are set to 0, we'll still bootstrap examples to use in our meta prompt
-# if (
-# max_bootstrapped_demos == 0 and max_labeled_demos == 0
-# ): # TODO: address case when max_bootstrapped alone is 0
-# max_bootstrapped_demos_for_candidate_gen = 1
-# max_labeled_demos_for_candidate_gen = 1 # TODO: this might only need to be 0
-# else:
-# max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos
-# max_labeled_demos_for_candidate_gen = max_labeled_demos
-
-# # Generate N few shot example sets
-# demo_candidates = {}
-# for i in range(self.num_candidates):
-# if i == 0: # Story empty set of demos as default for index 0
-# for module_p in module.predictors():
-# if id(module_p) not in demo_candidates:
-# demo_candidates[id(module_p)] = []
-# demo_candidates[id(module_p)].append([])
-# else:
-# if self.verbose:
-# print(f"Creating basic bootstrap: {i}/{self.num_candidates-1}")
-
-# # Create a new basic bootstrap few - shot program .
-# rng = random.Random(i)
-# shuffled_trainset = trainset[:] # Create a copy of devset
-# rng.shuffle(shuffled_trainset) # Shuffle the copy
-# tp = BootstrapFewShot(
-# metric=self.metric,
-# max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,
-# max_labeled_demos=max_labeled_demos_for_candidate_gen,
-# teacher_settings=self.teacher_settings,
-# )
-# candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_trainset)
-
-# # Store the candidate demos
-# for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):
-# if id(module_p) not in demo_candidates:
-# demo_candidates[id(module_p)] = []
-# demo_candidates[id(module_p)].append(candidate_p.demos)
-
-# # Generate N candidate prompts
-# instruction_candidates, _ = self._generate_first_N_candidates(
-# module,
-# self.num_candidates,
-# view_data,
-# view_examples,
-# demo_candidates,
-# trainset,
-# )
-
-# # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples
-# if max_bootstrapped_demos == 0 and max_labeled_demos == 0:
-# demo_candidates = None
-
-# # Initialize variables to store the best program and its score
-# best_score = float("-inf")
-# best_program = None
-# trial_num = 0
-
-# trial_logs = {}
-
-# # Define our trial objective
-# def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, trainset):
-# def objective(trial):
-# nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables
-# candidate_program = baseline_program.deepcopy()
-
-# # Suggest the instruction to use for our predictor
-# print(f"Starting trial #{trial_num}")
-# trial_logs[trial_num] = {}
-
-# for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):
-# # Get instruction candidates for our given predictor
-# p_instruction_candidates = instruction_candidates[id(p_old)]
-# if demo_candidates:
-# p_demo_candidates = demo_candidates[id(p_old)]
-
-# # Suggest the index of the instruction candidate to use in our trial
-# instruction_idx = trial.suggest_categorical(
-# f"{id(p_old)}_predictor_instruction",
-# range(len(p_instruction_candidates)),
-# )
-# if demo_candidates:
-# demos_idx = trial.suggest_categorical(
-# f"{id(p_old)}_predictor_demos",
-# range(len(p_demo_candidates)),
-# )
-# trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx
-# if demo_candidates:
-# trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx
-
-# # Get the selected instruction candidate
-# selected_candidate = p_instruction_candidates[instruction_idx]
-# selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()
-# selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()
-
-# # Use this candidates in our program
-# *_, last_field = self._get_signature(p_new).fields.keys()
-# updated_signature = (
-# self._get_signature(p_new)
-# .with_instructions(selected_instruction)
-# .with_updated_fields(last_field, prefix=selected_prefix)
-# )
-# self._set_signature(p_new, updated_signature)
-
-# # Get the selected demos
-# if demo_candidates:
-# selected_demos = p_demo_candidates[demos_idx]
-
-# # Use these demos in our program
-# if demo_candidates:
-# p_new.demos = selected_demos
-
-# if self.verbose:
-# print("Evaling the following program:")
-# if self.verbose:
-# self._print_full_program(candidate_program)
-# trial_logs[trial_num]["program"] = candidate_program
-
-# # Evaluate with the new prompts
-# total_score = 0
-# batch_size = 100
-# num_batches = math.ceil(len(trainset) / batch_size)
-
-# for i in range(num_batches):
-# start_index = i * batch_size
-# end_index = min((i + 1) * batch_size, len(trainset))
-# split_trainset = trainset[start_index:end_index]
-# split_score = evaluate(candidate_program, devset=split_trainset, display_table=0)
-# if self.verbose:
-# print(f"{i}st split score: {split_score}")
-
-# total_score += split_score * len(split_trainset)
-# curr_weighted_avg_score = total_score / min((i + 1) * 100, len(trainset))
-# if self.verbose:
-# print(f"curr average score: {curr_weighted_avg_score}")
-
-# trial.report(curr_weighted_avg_score, i)
-
-# # Handle pruning based on the intermediate value.
-# if trial.should_prune():
-# print("Trial pruned.")
-# trial_logs[trial_num]["score"] = curr_weighted_avg_score
-# trial_logs[trial_num]["pruned"] = True
-# trial_num += 1
-# raise optuna.TrialPruned()
-
-# if self.verbose:
-# print(f"Fully evaled score: {curr_weighted_avg_score}")
-# if self.verbose:
-# self._print_model_history(self.task_model, n=1)
-# score = curr_weighted_avg_score
-
-# trial_logs[trial_num]["score"] = curr_weighted_avg_score
-# trial_logs[trial_num]["pruned"] = False
-
-# # Update the best program if the current score is better
-# if score > best_score:
-# best_score = score
-# best_program = candidate_program.deepcopy()
-
-# trial_num += 1
-
-# return score
-
-# return objective
-
-# # Run the trial
-# objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)
-# sampler = optuna.samplers.TPESampler(seed=seed)
-# study = optuna.create_study(direction="maximize", sampler=sampler)
-# _score = study.optimize(objective_function, n_trials=num_trials)
-
-# if best_program is not None and self.track_stats:
-# best_program.trial_logs = trial_logs
-
-# print(f"Returning {best_program} from continue_program")
-# return best_program
-# return None
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.py
deleted file mode 100644
index 9f7a9b050..000000000
--- a/dspy/teleprompt/signature_opt_bayesian.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# from dspy.teleprompt.mipro_optimizer import MIPRO
-
-# """
-# ===============================================================
-# DEPRECATED!!!
-# PLEASE USE MIPRO INSTEAD.
-# ===============================================================
-
-# USAGE SUGGESTIONS:
-
-# The following code can be used to compile a optimized signature teleprompter using the BayesianSignatureOptimizer, and evaluate it on an end task:
-
-# from dspy.teleprompt import BayesianSignatureOptimizer
-
-# teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=10, init_temperature=1.0)
-# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
-# compiled_prompt_opt = teleprompter.compile(program, devset=devset[:DEV_NUM], optuna_trials_num=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
-# eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
-
-# Note that this teleprompter takes in the following parameters:
-
-# * prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
-# * task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
-# * metric: The task metric used for optimization.
-# * n: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.
-# * init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.
-# * verbose: Tells the method whether or not to print intermediate steps.
-# * track_stats: Tells the method whether or not to track statistics about the optimization process.
-# If True, the method will track a dictionary with a key corresponding to the trial number,
-# and a value containing a dict with the following keys:
-# * program: the program being evaluated at a given trial
-# * score: the last average evaluated score for the program
-# * pruned: whether or not this program was pruned
-# This information will be returned as attributes of the best program.
-# """
-
-
-# class BayesianSignatureOptimizer(MIPRO):
-# def __init__(
-# self,
-# prompt_model=None,
-# task_model=None,
-# teacher_settings={},
-# n=10,
-# metric=None,
-# init_temperature=1.0,
-# verbose=False,
-# track_stats=True,
-# view_data_batch_size=10,
-# ):
-# print(
-# "\u001b[31m[WARNING] BayesianSignatureOptimizer has been deprecated and replaced with MIPRO. BayesianSignatureOptimizer will be removed in a future release. \u001b[31m",
-# )
-
-# super().__init__(
-# metric=metric,
-# prompt_model=prompt_model,
-# task_model=task_model,
-# teacher_settings=teacher_settings,
-# num_candidates=n,
-# init_temperature=init_temperature,
-# verbose=verbose,
-# track_stats=track_stats,
-# view_data_batch_size=view_data_batch_size,
-# )
-
-# def compile(
-# self,
-# student,
-# *,
-# devset,
-# max_bootstrapped_demos,
-# max_labeled_demos,
-# eval_kwargs,
-# seed=42,
-# optuna_trials_num,
-# view_data=True,
-# view_examples=True,
-# requires_permission_to_run=False,
-# num_trials=None,
-# ):
-# return super().compile(
-# student,
-# trainset=devset,
-# max_bootstrapped_demos=max_bootstrapped_demos,
-# max_labeled_demos=max_labeled_demos,
-# eval_kwargs=eval_kwargs,
-# seed=seed,
-# view_data=view_data,
-# view_examples=view_examples,
-# requires_permission_to_run=requires_permission_to_run,
-# num_trials=optuna_trials_num,
-# )