DSPy: The framework for programming—not prompting—language models
Python
updates to mipro
klopsahlong07550964c0ed
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pyindex f7928ed17..82e5c78b4 100644--- a/dspy/adapters/chat_adapter.py+++ b/dspy/adapters/chat_adapter.py@@ -1,7 +1,7 @@import refrom .base import Adapter-field_header_pattern = re.compile(r'\[\[\[\[ #### (\w+) #### \]\]\]\]')+field_header_pattern = re.compile(r'\[\[\[ ### (\w+) ### \]\]\]')class ChatAdapter(Adapter):@@ -42,7 +42,7 @@ class ChatAdapter(Adapter):def format_fields(fields):- return '\n\n'.join([f"[[[[ #### {k} #### ]]]]\n{v}" for k, v in fields.items()]).strip()+ return '\n\n'.join([f"[[[ ### {k} ### ]]]\n{v}" for k, v in fields.items()]).strip()def format_chat_turn(field_names, values):if not set(values).issuperset(set(field_names)):diff --git a/dspy/propose/dataset_summary_generator.py b/dspy/propose/dataset_summary_generator.pyindex 6c6d5fcaf..7a4c03a96 100644--- a/dspy/propose/dataset_summary_generator.py+++ b/dspy/propose/dataset_summary_generator.py@@ -3,7 +3,6 @@ import reimport dspyfrom dspy.propose.utils import strip_prefix-class ObservationSummarizer(dspy.Signature):("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")observations = dspy.InputField(desc="Observations I have made about my dataset")@@ -46,7 +45,9 @@ def order_input_keys_in_string(unordered_repr):return ordered_reprdef create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None, verbose=False):+ if verbose: print("\nBootstrapping dataset summary (this will be used to generate instructions)...")upper_lim = min(len(trainset), view_data_batch_size)+ prompt_model = prompt_model if prompt_model else dspy.settings.lmwith dspy.settings.context(lm=prompt_model):observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=order_input_keys_in_string(trainset[0:upper_lim].__repr__()))observations = observation["observations"]@@ -78,10 +79,15 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_filexcept Exception as e:if verbose: print(f"e {e}. using observations from past round for a summary.")- with dspy.settings.context(lm=prompt_model):+ if prompt_model:+ with dspy.settings.context(lm=prompt_model):+ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)+ else:summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)if verbose: print(f"summary: {summary}")if log_file:log_file.write(f"summary: {summary}\n")++ if verbose: print(f"\nGenerated summary: {strip_prefix(summary.summary)}\n")return strip_prefix(summary.summary)diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.pyindex 1378d6ca9..0fa17a48d 100644--- a/dspy/propose/grounded_proposer.py+++ b/dspy/propose/grounded_proposer.py@@ -3,8 +3,8 @@ import reimport dspyfrom dspy.propose.dataset_summary_generator import create_dataset_summary-from dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix-from dspy.teleprompt.utils import get_signature+from dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix, get_dspy_source_code+from dspy.teleprompt.utils import get_signature, get_prompt_modelfrom .propose_base import Proposer@@ -186,21 +186,25 @@ class GenerateModuleInstruction(dspy.Module):program_description = ""module_code = ""if self.program_aware:- program_description = strip_prefix(- self.describe_program(- program_code=self.program_code_string, program_example=task_demos,- ).program_description,- )- if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")-- # Identify all modules- init_pattern = r"def __init__\([\s\S]*?\):([\s\S]*?)(?=^\s*def|\Z)"- init_content_match = re.search(init_pattern, self.program_code_string)- init_content = init_content_match.group(0)- pattern = r"^(.*dspy\.(ChainOfThought|Predict).*)$" # TODO: make it so that this extends out to any dspy Module- matches = re.findall(pattern, init_content, re.MULTILINE)- modules = [match[0].strip() for match in matches]- module_code = modules[pred_i]+ try:+ program_description = strip_prefix(+ self.describe_program(+ program_code=self.program_code_string, program_example=task_demos,+ ).program_description,+ )+ if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")++ # Identify all modules+ init_pattern = r"def __init__\([\s\S]*?\):([\s\S]*?)(?=^\s*def|\Z)"+ init_content_match = re.search(init_pattern, self.program_code_string)+ init_content = init_content_match.group(0)+ pattern = r"^(.*dspy\.(ChainOfThought|Predict).*)$" # TODO: make it so that this extends out to any dspy Module+ matches = re.findall(pattern, init_content, re.MULTILINE)+ modules = [match[0].strip() for match in matches]+ module_code = modules[pred_i]+ except:+ if self.verbose: print("Error getting program description. Running without program aware proposer.")+ self.program_aware = Falsemodule_description = self.describe_module(program_code=self.program_code_string,@@ -236,8 +240,8 @@ class GroundedProposer(Proposer):def __init__(self,prompt_model,+ program,trainset,- program_code_string=None,view_data_batch_size=10,use_dataset_summary=True,program_aware=True,@@ -258,8 +262,17 @@ class GroundedProposer(Proposer):self.set_history_randomly=set_history_randomlyself.verbose = verbose- self.prompt_model = prompt_model- self.program_code_string = program_code_string+ self.prompt_model = get_prompt_model(prompt_model)+ if self.program_aware:+ try:+ self.program_code_string = get_dspy_source_code(program)+ if self.verbose: print("SOURCE CODE:",self.program_code_string)+ except Exception as e:+ print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")+ self.program_code_string = None+ self.program_aware = False+ else:+ self.program_code_string = Noneself.data_summary = create_dataset_summary(trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,)@@ -270,7 +283,6 @@ class GroundedProposer(Proposer):trainset,program,demo_candidates,- prompt_model,trial_logs,N,T,@@ -306,7 +318,7 @@ class GroundedProposer(Proposer):program=program,predictor=predictor,pred_i=pred_i,- prompt_model=prompt_model,+ # prompt_model=prompt_model,T=T,demo_candidates=demo_candidates,demo_set_i=demo_set_i,@@ -321,7 +333,7 @@ class GroundedProposer(Proposer):program,predictor,pred_i,- prompt_model,+ # prompt_model,T,demo_candidates,demo_set_i,@@ -347,9 +359,10 @@ class GroundedProposer(Proposer):)# Generate a new instruction for our predictor, using the temperature specified for this round- original_temp = prompt_model.kwargs["temperature"]- with dspy.settings.context(lm=prompt_model):- prompt_model.kwargs["temperature"] = T+ original_temp = self.prompt_model.kwargs["temperature"]++ with dspy.settings.context(lm=self.prompt_model):+ self.prompt_model.kwargs["temperature"] = Tproposed_instruction = instruction_generator.forward(demo_candidates=demo_candidates,pred_i=pred_i,@@ -359,10 +372,10 @@ class GroundedProposer(Proposer):previous_instructions=instruction_history,tip=tip,).proposed_instruction- prompt_model.kwargs["temperature"] = original_temp+ self.prompt_model.kwargs["temperature"] = original_temp# Log the trace used to generate the new instruction, along with the new instruction itself- prompt_model.inspect_history(n=1)+ if self.verbose: self.prompt_model.inspect_history(n=1)if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")return strip_prefix(proposed_instruction)diff --git a/dspy/propose/utils.py b/dspy/propose/utils.pyindex 736e1b925..99e146f71 100644--- a/dspy/propose/utils.py+++ b/dspy/propose/utils.py@@ -1,8 +1,16 @@import jsonimport re+import dspy+import inspect+try:+ from IPython.core.magics.code import extract_symbols+except ImportError:+ # Won't be able to read code from juptyer notebooks+ extract_symbols = None-from dspy.teleprompt.utils import get_signature+from dspy.predict.parameter import Parameter+from dspy.teleprompt.utils import get_signaturedef strip_prefix(text):pattern = r'^[\*\s]*(([\w\'\-]+\s+){0,4}[\w\'\-]+):\s*'@@ -131,4 +139,45 @@ def create_example_string(fields, example):output.append(field_str)# Joining all the field strings- return '\n'.join(output)\ No newline at end of file+ return '\n'.join(output)++def get_dspy_source_code(module):+ header = []+ base_code = ""++ # Don't get source code for Predict or ChainOfThought modules (NOTE we will need to extend this list as more DSPy.modules are added)+ if not type(module).__name__ == "Predict" and not type(module).__name__ == "ChainOfThought":+ try:+ base_code = inspect.getsource(type(module))+ except TypeError:+ obj = type(module)+ cell_code = "".join(inspect.linecache.getlines(new_getfile(obj)))+ class_code = extract_symbols(cell_code, obj.__name__)[0][0]+ base_code = str(class_code)++ completed_set = set()+ for attribute in module.__dict__.keys():+ try:+ iterable = iter(getattr(module, attribute))+ except TypeError:+ iterable = [getattr(module, attribute)]++ for item in iterable:+ if item in completed_set:+ continue+ if isinstance(item, Parameter):+ if hasattr(item, 'signature') and item.signature is not None and item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig" not in completed_set:+ try:+ header.append(inspect.getsource(item.signature))+ print(inspect.getsource(item.signature))+ except (TypeError, OSError):+ header.append(str(item.signature))+ completed_set.add(item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig")+ if isinstance(item, dspy.Module):+ code = get_dspy_source_code(item).strip()+ if code not in completed_set:+ header.append(code)+ completed_set.add(code)+ completed_set.add(item)++ return '\n\n'.join(header) + '\n\n' + base_code\ No newline at end of filediff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex 18239d98f..95e04bc47 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -8,6 +8,7 @@ from collections import defaultdictimport numpy as npimport optuna+import dspyfrom dspy.evaluate.evaluate import Evaluatefrom dspy.propose import GroundedProposer@@ -15,7 +16,6 @@ from dspy.teleprompt.teleprompt import Teleprompterfrom dspy.teleprompt.utils import (create_n_fewshot_demo_sets,eval_candidate_program,- get_dspy_source_code,get_program_with_highest_avg_score,get_signature,get_task_model_history_for_full_example,@@ -23,7 +23,7 @@ from dspy.teleprompt.utils import (save_candidate_program,save_file_to_log_dir,set_signature,- setup_logging,+ # setup_logging,)try:@@ -41,7 +41,7 @@ from dspy.teleprompt import MIPROv2teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_batches=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)```@@ -73,100 +73,86 @@ Note that this teleprompter takes in the following parameters:BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0-MB_FULL_EVAL_STEPS = 10-MINIBATCH_SIZE = 25#50-class MIPROv2(Teleprompter):def __init__(self,+ metric,prompt_model=None,task_model=None,teacher_settings={},num_candidates=10,- metric=None,- init_temperature=1.4,+ num_threads=6,+ max_errors=10,+ init_temperature=0.5,verbose=False,track_stats=True,log_dir=None,- view_data_batch_size=10,- minibatch_size=MINIBATCH_SIZE,- minibatch_full_eval_steps=MB_FULL_EVAL_STEPS,metric_threshold=None,):- self.n = num_candidates+ self.num_candidates = num_candidatesself.metric = metricself.init_temperature = init_temperature- self.prompt_model = prompt_model- self.task_model = task_model+ self.prompt_model = prompt_model if prompt_model else dspy.settings.lm+ self.task_model = task_model if task_model else dspy.settings.lmself.verbose = verboseself.track_stats = track_statsself.log_dir = log_dir- self.view_data_batch_size = view_data_batch_sizeself.teacher_settings = teacher_settingsself.prompt_model_total_calls = 0self.total_calls = 0- self.minibatch_size = minibatch_size- self.minibatch_full_eval_steps = minibatch_full_eval_steps- self.metric_threshold = None+ self.num_threads = num_threads+ self.max_errors = max_errors+ self.metric_threshold = metric_threshold# Check if WANDB_RUN_ID is set in the environmentself.wandb_run_id = None-- def _get_batch_size(- self,- minibatch,- trainset,- ):- if minibatch:- return self.minibatch_size- else:- return len(trainset)def compile(self,student,*,trainset,- valset=None,- num_batches=30,- max_bootstrapped_demos=5,- max_labeled_demos=2,- eval_kwargs={},+ num_trials=30,+ max_bootstrapped_demos=4,+ max_labeled_demos=16,seed=9,minibatch=True,program_aware_proposer=True,requires_permission_to_run=True,+ view_data_batch_size=10,+ minibatch_size=25, # TODO: more intelligently set these defaults+ minibatch_full_eval_steps=10,):# Define ANSI escape codes for colorsYELLOW = "\033[93m"+ GREEN = "\033[92m"BLUE = "\033[94m"BOLD = "\033[1m"ENDC = "\033[0m" # Resets the color to defaultrandom.seed(seed)- valset = valset or trainset- estimated_prompt_model_calls = 10 + self.n * len(+ estimated_prompt_model_calls = 10 + self.num_candidates * len(student.predictors(),) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)prompt_model_line = ""if not program_aware_proposer:- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""else:- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""estimated_task_model_calls_wo_module_calls = 0task_model_line = ""if not minibatch:- estimated_task_model_calls_wo_module_calls = len(trainset) * num_batches # M * T * P- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""+ estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""else:- estimated_task_model_calls_wo_module_calls = self.minibatch_size * num_batches + (len(trainset) * (num_batches // self.minibatch_full_eval_steps)) # B * T * P- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{self.minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches // self.minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""+ estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (len(trainset) * (num_trials // minibatch_full_eval_steps)) # B * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_trials // minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""user_message = textwrap.dedent(f"""\- {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}+ {YELLOW}{BOLD}Projected Language Model (LM) Calls{ENDC}Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:@@ -182,22 +168,23 @@ class MIPROv2(Teleprompter):For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the taskand prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:- {YELLOW}- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.{ENDC}- {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")+ {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n""")user_confirmation_message = textwrap.dedent(f"""\To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.- If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}+ If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.{YELLOW}Awaiting your input...{ENDC}""")- print(user_message)+ if requires_permission_to_run: print(user_message)sys.stdout.flush() # Flush the output buffer to force the message to printrun = True+ # TODO: make sure these estimates are good for mini-batchingif requires_permission_to_run:print(user_confirmation_message)user_input = input("Do you wish to continue? (y/n): ").strip().lower()@@ -206,53 +193,20 @@ class MIPROv2(Teleprompter):run = Falseif run:- if program_aware_proposer:- try:- self.program_code_string = get_dspy_source_code(student)- if self.verbose:- if self.verbose: print("SOURCE CODE:",self.program_code_string)- except Exception as e:- print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")- self.program_code_string = None- program_aware_proposer = False- else:- self.program_code_string = None-- # Setup our proposer- proposer = GroundedProposer(- trainset=trainset,- prompt_model=self.prompt_model,- program_code_string=self.program_code_string,- view_data_batch_size=self.view_data_batch_size,- program_aware=program_aware_proposer,- )-- # Setup logging- logging.getLogger('openai').setLevel(logging.WARNING)- logging.basicConfig(level=logging.WARNING)- setup_logging(self.log_dir)-- self.wandb_run_id = os.getenv("WANDB_RUN_ID", None)-- if self.wandb_run_id:- # Initialize wandb with the same run ID- wandb.init(- project="prompt_optimizers",- id=self.wandb_run_id,- resume="must",- )-# Setup random seedsrandom.seed(seed)np.random.seed(seed)- # Log current file to log_dir- curr_file = os.path.abspath(__file__)- save_file_to_log_dir(curr_file, self.log_dir)-# Set up program and evaluation functionprogram = student.deepcopy()- evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)+ evaluate = Evaluate(+ devset=trainset,+ metric=self.metric,+ num_threads=self.num_threads,+ max_errors=self.max_errors,+ display_table=False,+ display_progress=True,+ )# Determine the number of fewshot examples to use to generate demos for promptif max_bootstrapped_demos == 0 and max_labeled_demos == 0:@@ -261,20 +215,24 @@ class MIPROv2(Teleprompter):else:max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demosmax_labeled_demos_for_candidate_gen = max_labeled_demos-- # # Generate N few shot example sets- # if not demo_candidates and not instruction_candidates:- # demo_candidates = create_n_fewshot_example_sets(program=program, trainset=trainset, n=self.n, hard_fewshot=hard_fewshot, metric=self.metric, teacher_settings=self.teacher_settings, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen)-- # Generate N few shot example sets++ # Generate N few shot example sets (these will inform instruction creation, and be used as few-shot examples in our prompt)+ print(f"Beginning MIPROv2 optimization process...")+ print(f"\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")+ if max_bootstrapped_demos > 0:+ print(f"These will be used for as few-shot examples candidates for our program and for creating instructions.\n")+ else:+ print(f"These will be used for informing instruction proposal.\n")+ print(f"Bootstrapping N={self.num_candidates} sets of demonstrations...")try:demo_candidates = create_n_fewshot_demo_sets(student=program,- num_candidate_sets=self.n,+ num_candidate_sets=self.num_candidates,trainset=trainset,max_labeled_demos=max_labeled_demos_for_candidate_gen,max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,metric=self.metric,+ max_errors=self.max_errors,teacher_settings=self.teacher_settings,seed=seed,metric_threshold=self.metric_threshold,@@ -283,38 +241,48 @@ class MIPROv2(Teleprompter):print(f"Error generating fewshot examples: {e}")print("Running without fewshot examples.")demo_candidates = None++ # Generate N candidate instructions- # Generate N candidate prompts- proposer.program_aware = program_aware_proposer- proposer.use_tip = True- proposer.use_instruct_history = False- proposer.set_history_randomly = False+ # Setup our proposer+ print(f"\n==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==")+ print("In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, and a summary of the program code, and a randomly selected prompting tip to propose instructions.")++ proposer = GroundedProposer(+ program=program,+ trainset=trainset,+ prompt_model=self.prompt_model,+ # program_code_string=self.program_code_string,+ view_data_batch_size=view_data_batch_size,+ program_aware=program_aware_proposer,+ use_instruct_history=False,+ set_history_randomly=False,+ verbose = self.verbose,+ )++ print(f"\nProposing instructions...\n")instruction_candidates = proposer.propose_instructions_for_program(trainset=trainset,program=program,demo_candidates=demo_candidates,- N=self.n,- prompt_model=self.prompt_model,+ N=self.num_candidates,T=self.init_temperature,trial_logs={},)for i, pred in enumerate(program.predictors()):+ print(f"Proposed Instructions for Predictor {i}:\n")instruction_candidates[i][0] = get_signature(pred).instructions+ for j, instruction in enumerate(instruction_candidates[i]):+ print(f"{j}: {instruction}\n")+ print("\n")- # instruction_candidates[1][0] = "Given the question, and context, respond with the number of the document that is most relevant to answering the question in the field 'Answer' (ex. Answer: '3')."-- # Save the candidate instructions generated- if self.log_dir:- fp = os.path.join(self.log_dir, "instructions_to_save.pickle")- with open(fp, "wb") as file:- pickle.dump(instruction_candidates, file)-- # If we're doing zero-shot, reset demo_candidates to none+ # If we're doing zero-shot, reset demo_candidates to none now that we've used them for instruction proposalif max_bootstrapped_demos == 0 and max_labeled_demos == 0:demo_candidates = None- # Initialize variables to store the best program and its score+ # Initialize variables to track during the optimization processbest_score = float("-inf")+ best_scoring_trial = 0best_program = Nonetrial_logs = {}total_eval_calls = 0@@ -330,11 +298,14 @@ class MIPROv2(Teleprompter):trainset,):def objective(trial):- nonlocal best_program, best_score, trial_logs, total_eval_calls # Allow access to the outer variables+ nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls # Allow access to the outer variables# Kick off trial- logging.info(f"Starting trial num: {trial.number}")- trial_logs[trial.number] = {}+ if minibatch:+ print(f"== Minibatch Trial {trial.number+1} / {num_trials}==")+ else:+ print(f"===== Trial {trial.number+1} / {num_trials} =====")+ trial_logs[trial.number+1] = {}# Create a new candidate programcandidate_program = baseline_program.deepcopy()@@ -353,23 +324,24 @@ class MIPROv2(Teleprompter):f"{i}_predictor_instruction",range(len(p_instruction_candidates)),)- chosen_params.append(instruction_idx)+ # chosen_params.append(instruction_idx)+ chosen_params.append(f"Predictor {i+1}: Instruction {instruction_idx}")if demo_candidates:demos_idx = trial.suggest_categorical(f"{i}_predictor_demos", range(len(p_demo_candidates)),)- chosen_params.append(demo_candidates)+ chosen_params.append(f"Predictor {i+1}: Few-Shot Set {demos_idx}")# Log the selected instruction / demo candidate- trial_logs[trial.number][+ trial_logs[trial.number+1][f"{i}_predictor_instruction"] = instruction_idxif demo_candidates:- trial_logs[trial.number][f"{i}_predictor_demos"] = demos_idx+ trial_logs[trial.number+1][f"{i}_predictor_demos"] = demos_idx- logging.info(f"instruction_idx {instruction_idx}")+ dspy.logger.debug(f"instruction_idx {instruction_idx}")if demo_candidates:- logging.info(f"demos_idx {demos_idx}")+ dspy.logger.debug(f"demos_idx {demos_idx}")# Set the instructionselected_instruction = p_instruction_candidates[instruction_idx]@@ -383,119 +355,115 @@ class MIPROv2(Teleprompter):p_new.demos = p_demo_candidates[demos_idx]# Log assembled program- if self.verbose: print("CANDIDATE PROGRAM:")+ if self.verbose: print("Evaluating the following candidate program...\n")if self.verbose: print_full_program(candidate_program)- if self.verbose: print("...")# Save the candidate program- trial_logs[trial.number]["program_path"] = save_candidate_program(- candidate_program, self.log_dir, trial.number,+ trial_logs[trial.number+1]["program_path"] = save_candidate_program(+ candidate_program, self.log_dir, trial.number+1,)- trial_logs[trial.number]["num_eval_calls"] = 0+ trial_logs[trial.number+1]["num_eval_calls"] = 0# Evaluate the candidate program with relevant batch size- batch_size = self._get_batch_size(minibatch, trainset)+ batch_size = minibatch_size if minibatch else len(trainset)score = eval_candidate_program(batch_size, trainset, candidate_program,evaluate,)- if self.verbose: print(f"SCORE {score}, BATCH SIZE {batch_size}")# Print out a full trace of the program in useif self.verbose:- print("FULL TRACE")+ print("Full trace of prompts in use on an example...")get_task_model_history_for_full_example(candidate_program, self.task_model, trainset, evaluate,)- print("...")# Log relevant information- if self.verbose: print(f"Score {score}")categorical_key = ",".join(map(str, chosen_params))param_score_dict[categorical_key].append((score, candidate_program),)- trial_logs[trial.number]["num_eval_calls"] = batch_size- trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)- trial_logs[trial.number]["score"] = score- trial_logs[trial.number]["pruned"] = False- total_eval_calls += trial_logs[trial.number]["num_eval_calls"]- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number]["program"] = candidate_program.deepcopy()- if self.wandb_run_id:- wandb.log(- {- "score": score,- "num_eval_calls": trial_logs[trial.number]["num_eval_calls"],- "total_eval_calls": total_eval_calls,- },- )-- # Update the best program if the current score is better, and if we're not using minibatching+ trial_logs[trial.number+1]["num_eval_calls"] = batch_size+ trial_logs[trial.number+1]["full_eval"] = batch_size >= len(trainset)+ trial_logs[trial.number+1]["score"] = score+ trial_logs[trial.number+1]["pruned"] = False+ total_eval_calls += trial_logs[trial.number+1]["num_eval_calls"]+ trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number+1]["program"] = candidate_program.deepcopy()++ # If this score was from a full evaluation, update the best program if the new score is betterbest_score_updated = False- if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:- if self.verbose: print("Updating best score")+ if score > best_score and trial_logs[trial.number+1]["full_eval"] and not minibatch:best_score = score+ best_scoring_trial = trial.number+1best_program = candidate_program.deepcopy()best_score_updated = True++ if minibatch:+ print(f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n")+ else:+ print(f"Score: {score} with parameters {chosen_params}.")+ if best_score_updated:+ print(f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n")+ else:+ print(f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n")# If we're doing minibatching, check to see if it's time to do a full eval- if minibatch and trial.number % self.minibatch_full_eval_steps == 0:+ if minibatch and trial.number+1 % minibatch_full_eval_steps == 0:+ print(f"===== Full Eval {trial.number+1 // minibatch_full_eval_steps} =====")# Save old information as the minibatch version- trial_logs[trial.number]["mb_score"] = score- trial_logs[trial.number]["mb_program_path"] = trial_logs[trial.number]["program_path"]+ trial_logs[trial.number+1]["mb_score"] = score+ trial_logs[trial.number+1]["mb_program_path"] = trial_logs[trial.number+1]["program_path"]# Identify our best program (based on mean of scores so far, and do a full eval on it)- highest_mean_program, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)+ highest_mean_program, mean, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)full_train_score = eval_candidate_program(len(trainset), trainset, highest_mean_program, evaluate,)+ if trial.number+1 // minibatch_full_eval_steps > 0:+ print(f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials...")+ else:+ print(f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials...")+# Log relevant informationfully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}total_eval_calls += len(trainset)- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number]["full_eval"] = True- trial_logs[trial.number]["program_path"] = save_candidate_program(- program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number, note="full_eval",+ trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number+1]["full_eval"] = True+ trial_logs[trial.number+1]["program_path"] = save_candidate_program(+ program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number+1, note="full_eval",)- trial_logs[trial.number]["score"] = full_train_score+ trial_logs[trial.number+1]["score"] = full_train_scoreif full_train_score > best_score:- if self.verbose: print(f"UPDATING BEST SCORE WITH {full_train_score}")+ print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_train_score}")best_score = full_train_score+ best_scoring_trial = trial.number+1best_program = highest_mean_program.deepcopy()best_score_updated = True+ else:+ print(f"Full eval score: {full_train_score}")+ print(f"Best full eval score so far: {best_score}")+ print("=======================\n\n")- # If the best score was updated, do a full eval on the dev set- if best_score_updated:- full_dev_score = evaluate(- best_program,- devset=valset,- display_table=0,- )- if self.wandb_run_id:- wandb.log(- {- "best_prog_so_far_train_score": best_score,- "best_prog_so_far_dev_score": full_dev_score,- },- )-return scorereturn objective# Run the trial+ optuna.logging.set_verbosity(optuna.logging.WARNING)+ print(f"\n==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==")+ print("In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n")objective_function = create_objective(program, instruction_candidates, demo_candidates, evaluate, trainset,)sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)study = optuna.create_study(direction="maximize", sampler=sampler)- _ = study.optimize(objective_function, n_trials=num_batches)+ _ = study.optimize(objective_function, n_trials=num_trials)if best_program is not None and self.track_stats:best_program.trial_logs = trial_logs@@ -503,13 +471,6 @@ class MIPROv2(Teleprompter):best_program.prompt_model_total_calls = self.prompt_model_total_callsbest_program.total_calls = self.total_calls- if self.log_dir:- program_file_path = os.path.join(self.log_dir, "best_program")- best_program.save(program_file_path)-- optuna_study_file_path = os.path.join(self.log_dir, "optuna_study.pickle")- with open(optuna_study_file_path, "wb") as file:- pickle.dump(study, file)-return best_program+return studentdiff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pyindex 0d7000293..7973f9b89 100644--- a/dspy/teleprompt/utils.py+++ b/dspy/teleprompt/utils.py@@ -44,13 +44,12 @@ def eval_candidate_program(batch_size, trainset, candidate_program, evaluate):"""Evaluate a candidate program on the trainset, using the specified batch size."""# Evaluate on the full trainsetif batch_size >= len(trainset):- score = evaluate(candidate_program, devset=trainset, display_table=0)+ score = evaluate(candidate_program, devset=trainset)# Or evaluate on a minibatchelse:score = evaluate(candidate_program,devset=create_minibatch(trainset, batch_size),- display_table=0,)return score@@ -120,12 +119,10 @@ def get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combif key in fully_evaled_param_combos:continue- print(f"Best Combination: {key} with Mean = {mean}")-- return program, key+ return program, mean, key# If no valid program is found, we return the last valid one that we found- return program, key+ return program, mean, keydef calculate_last_n_proposed_quality(@@ -184,7 +181,7 @@ def print_full_program(program):print(f"i: {get_signature(predictor).instructions}")*_, last_field = get_signature(predictor).fields.values()print(f"p: {last_field.json_schema_extra['prefix']}")- print("\n")+ print("\n")def save_candidate_program(program, log_dir, trial_num, note=None):@@ -246,6 +243,12 @@ def setup_logging(log_dir):### OTHER UTILS ###+def get_prompt_model(prompt_model):+ if prompt_model:+ return prompt_model+ else:+ return dspy.settings.lm+def get_signature(predictor):if hasattr(predictor, "extended_signature"):return predictor.extended_signature@@ -269,6 +272,7 @@ def create_n_fewshot_demo_sets(max_bootstrapped_demos,metric,teacher_settings,+ max_errors=10,max_rounds=1,labeled_sample=True,min_num_samples=1,@@ -297,6 +301,8 @@ def create_n_fewshot_demo_sets(# Go through and create each candidate setfor seed in range(-3, num_candidate_sets):+ print(f"Bootstrapping set {seed+4}/{num_candidate_sets+3}")+trainset2 = list(trainset)if seed == -3 and include_non_bootstrapped:@@ -318,6 +324,7 @@ def create_n_fewshot_demo_sets(# unshuffled few-shotprogram = BootstrapFewShot(metric=metric,+ max_errors=max_errors,max_bootstrapped_demos=max_bootstrapped_demos,max_labeled_demos=max_labeled_demos,teacher_settings=teacher_settings,@@ -332,6 +339,7 @@ def create_n_fewshot_demo_sets(teleprompter = BootstrapFewShot(metric=metric,+ max_errors=max_errors,metric_threshold=metric_threshold,max_bootstrapped_demos=size,max_labeled_demos=max_labeled_demos,@@ -392,45 +400,4 @@ def new_getfile(object):return inspect.getfile(member)raise TypeError(f'Source for {object!r} not found')-inspect.getfile = new_getfile--def get_dspy_source_code(module):- header = []- base_code = ""-- # Don't get source code for Predict or ChainOfThought modules (NOTE we will need to extend this list as more DSPy.modules are added)- if not type(module).__name__ == "Predict" and not type(module).__name__ == "ChainOfThought":- try:- base_code = inspect.getsource(type(module))- except TypeError:- obj = type(module)- cell_code = "".join(inspect.linecache.getlines(new_getfile(obj)))- class_code = extract_symbols(cell_code, obj.__name__)[0][0]- base_code = str(class_code)-- completed_set = set()- for attribute in module.__dict__.keys():- try:- iterable = iter(getattr(module, attribute))- except TypeError:- iterable = [getattr(module, attribute)]-- for item in iterable:- if item in completed_set:- continue- if isinstance(item, Parameter):- if hasattr(item, 'signature') and item.signature is not None and item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig" not in completed_set:- try:- header.append(inspect.getsource(item.signature))- print(inspect.getsource(item.signature))- except (TypeError, OSError):- header.append(str(item.signature))- completed_set.add(item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig")- if isinstance(item, dspy.Module):- code = get_dspy_source_code(item).strip()- if code not in completed_set:- header.append(code)- completed_set.add(code)- completed_set.add(item)-- return '\n\n'.join(header) + '\n\n' + base_code\ No newline at end of file+inspect.getfile = new_getfile\ No newline at end of filediff --git a/examples/optimizers/mipro_test.py b/examples/optimizers/mipro_test.pynew file mode 100644index 000000000..bcca24f9a--- /dev/null+++ b/examples/optimizers/mipro_test.py@@ -0,0 +1,118 @@+import dspy+import os;+os.environ['LITELLM_LOG'] = 'INFO'+from datetime import datetime++# lm = dspy.LM("gpt-4o-mini-2024-07-18", api_base=os.getenv('OPENAI_API_BASE'), api_key=os.getenv('OPENAI_API_KEY'))+# lm = dspy.HFClientVLLM(+# model="meta-llama/Meta-Llama-3-8B",+# port=7410,+# url=["http://future-hgx-1:7501"],+# max_tokens=250,+# stop=["\n\n", "\n---", "assistant"],+# )++# lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY", temperature=0.4, max_tokens=500)+lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY", max_tokens=700)+# lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY")+++dspy.configure(lm=lm)++# dspy.ChainOfThought('question -> answer: int')(question="What is two plus two?")+# breakpoint()++from dspy.datasets.gsm8k import GSM8K, gsm8k_metric++gms8k = GSM8K()++trainset, devset = gms8k.train, gms8k.dev++program = dspy.ChainOfThought('question -> answer: int')++from dspy.evaluate import Evaluate++evaluate = Evaluate(devset=devset[:100], metric=gsm8k_metric, num_threads=1, display_progress=True, display_table=False, max_errors=10000)++# class GSM8Reflection(dspy.Module):+# def __init__(self, n = 3):+# super().__init__()+# self.gen_candidate_answer = dspy.ChainOfThought("question -> answer:int", n=n)+# self.gen_final_answer = dspy.ChainOfThought("candidate_answers -> final_answer:int")++# def get_candidate_answers_string(self, candidate_answers):+# return "\n".join([f"Potential Answer {i+1}:\n\nReasoning: Let's think step by step to {candidate_answers.completions.rationale[i]}\n\nAnswer: {candidate_answers.completions.answer[i]}\n\n" for i in range(len(candidate_answers.completions.answer))])++# def forward(self, question):+# candidate_answers = self.gen_candidate_answer(question=question)+# create_candidate_answer_string = self.get_candidate_answers_string(candidate_answers=candidate_answers)+# final_answer = self.gen_final_answer(candidate_answers=create_candidate_answer_string).final_answer+# return dspy.Prediction(answer=final_answer)++# program = GSM8Reflection()+print(f"Evaluating baseline program...")+evaluate(program, devset=devset[:300])+breakpoint()+# Apply 0-shot MIPRO+from dspy.teleprompt import MIPROv2++# teleprompter = MIPROv2(+# metric=gsm8k_metric,+# num_candidates=10,+# max_errors = 10000,+# init_temperature=0.5,+# verbose=False,+# num_threads=4,+# # log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",+# )++# zeroshot_optimized_program = teleprompter.compile(+# program.deepcopy(),+# trainset=trainset[:200],+# max_bootstrapped_demos=0, # 0-shot optimization+# max_labeled_demos=0,+# num_batches=10,+# # minibatch_size=25,+# # minibatch_full_eval_steps=10, # TODO: should these be params in the compile step instead?+# minibatch=False,+# seed=9,+# requires_permission_to_run=False,+# )++# now = datetime.now()+# date_time = now.strftime("%Y%m%d_%H%M%S")+# zeroshot_optimized_program.save(f"zeroshot_mipro_optimized_{date_time}")++# print(f"Evluate optimized program...")+# evaluate(zeroshot_optimized_program, devset=devset[:300])+++teleprompter = MIPROv2(+ metric=gsm8k_metric,+ num_candidates=10,+ max_errors = 10000,+ init_temperature=0.5,+ verbose=False,+ num_threads=4,+ # log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",+)++optimized_program = teleprompter.compile(+ program.deepcopy(),+ trainset=trainset[:200],+ max_bootstrapped_demos=3, # 0-shot optimization+ max_labeled_demos=4,+ num_trials=15,+ # minibatch_size=25,+ # minibatch_full_eval_steps=10, # TODO: should these be params in the compile step instead?+ minibatch=False,+ seed=9,+ requires_permission_to_run=False,+)++now = datetime.now()+date_time = now.strftime("%Y%m%d_%H%M%S")+optimized_program.save(f"mipro_optimized_{date_time}")++print(f"Evaluate optimized program...")+evaluate(optimized_program, devset=devset[:300])\ No newline at end of file