Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

refactoring optimizers, updates to optimizer notebook

klopsahlong610ad5966eec
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.py
index a1088f2df..ba14c1b94 100644
--- a/dspy/teleprompt/__init__.py
+++ b/dspy/teleprompt/__init__.py
@@ -1,10 +1,11 @@
+from .teleprompt import *
from .bootstrap import *
-from .ensemble import *
+from .vanilla import *
+from .random_search import *
from .finetune import *
+from .teleprompt_optuna import *
from .knn_fewshot import *
-from .random_search import *
from .signature_opt import SignatureOptimizer
from .signature_opt_bayesian import BayesianSignatureOptimizer
-from .teleprompt import *
-from .teleprompt_optuna import *
-from .vanilla import *
+from .mipro_optimizer import MIPRO
+from .copro_optimizer import COPRO
\ No newline at end of file
diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.py
new file mode 100644
index 000000000..f1c303a47
--- /dev/null
+++ b/dspy/teleprompt/copro_optimizer.py
@@ -0,0 +1,301 @@
+from collections import defaultdict
+
+import dsp
+import dspy
+from dspy.evaluate.evaluate import Evaluate
+from dspy.signatures import Signature
+from dspy.teleprompt.teleprompt import Teleprompter
+
+"""
+USAGE SUGGESTIONS:
+
+The following code can be used to compile a optimized signature teleprompter, and evaluate it on an end task:
+
+teleprompter = COPRO(prompt_model=prompt_model, metric=metric, breadth=BREADTH, depth=DEPTH, init_temperature=INIT_TEMPERATURE)
+kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
+compiled_prompt_opt = teleprompter.compile(program.deepcopy(), devset=devset[:DEV_NUM], eval_kwargs=kwargs)
+eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
+
+Note that this teleprompter takes in the following parameters:
+
+* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
+* metric: The task metric used for optimization.
+* breadth: The number of new prompts to generate at each iteration. Default=10.
+* depth: The number of times we should ask our prompt model to generate new prompts, with the history of the past prompts as input. Default=3.
+* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.4.
+* verbose: Tells the method whether or not to print intermediate steps.
+* track_stats: Tells the method whether or not to track statistics about the optimization process.
+ If True, the method will track the following statistics:
+ * results_best: The min,max,avg,stddev of top 10 scores for each predictor at each depth.
+ * results_latest: The min,max,avg,stddev of newest prompt scores for each predictor at each depth.
+ * total_calls: The total number of calls to the task metric.
+ These statistics will be returned as attributes of the best program.
+"""
+class BasicGenerateInstruction(Signature):
+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
+
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class GenerateInstructionGivenAttempts(dspy.Signature):
+ """You are an instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality.
+
+Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative."""
+
+ attempted_instructions = dspy.InputField(format=dsp.passages2text)
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class COPRO(Teleprompter):
+ def __init__(self, prompt_model=None, metric=None, breadth=10, depth=3, init_temperature=1.4, verbose=False, track_stats=False):
+ if breadth <= 1:
+ raise ValueError("Breadth must be greater than 1")
+ self.metric = metric
+ self.breadth = breadth
+ self.depth = depth
+ self.init_temperature = init_temperature
+ self.prompt_model = prompt_model
+ self.verbose = verbose
+ self.track_stats = track_stats
+
+ def _check_candidates_equal(self, candidate1, candidate2):
+ for p1, p2 in zip(candidate1["program"].predictors(), candidate2["program"].predictors()):
+ if self._get_signature(p1).instructions != self._get_signature(p2).instructions:
+ return False
+ *_, p1_last_field = self._get_signature(p1).fields.values()
+ *_, p2_last_field = self._get_signature(p2).fields.values()
+ if p1_last_field != p2_last_field:
+ return False
+ return True
+
+ def _drop_duplicates(self, candidates):
+ final_candidates = []
+ last_batch = []
+ last_batch_score = -1
+ for c in candidates:
+ repeat = False
+ if c['score'] == last_batch_score:
+ for c2 in last_batch:
+ if (self._check_candidates_equal(c, c2)):
+ repeat = True
+ break
+ if not repeat:
+ last_batch.append(c)
+ else:
+ last_batch = [c]
+ last_batch_score = c['score']
+ if not repeat:
+ final_candidates.append(c)
+ return final_candidates
+
+ def _print_signature(self, predictor):
+ if self.verbose:
+ signature = self._get_signature(predictor)
+ print(f"i: {signature.instructions}")
+ print(f"p: {list(signature.fields.values())[-1].json_schema_extra['prefix']}")
+ print()
+
+ def _get_signature(self, predictor):
+ if (hasattr(predictor, 'extended_signature')):
+ return predictor.extended_signature
+ elif (hasattr(predictor, 'signature')):
+ return predictor.signature
+
+ def _set_signature(self, predictor, updated_signature):
+ if (hasattr(predictor, 'extended_signature')):
+ predictor.extended_signature = updated_signature
+ elif (hasattr(predictor, 'signature')):
+ predictor.signature = updated_signature
+
+
+ def compile(self, student, *, devset, eval_kwargs):
+ """student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""
+ module = student.deepcopy()
+ evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
+ total_calls = 0
+ results_best = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}
+ results_latest = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}
+
+ if self.track_stats:
+ import numpy as np
+
+
+ candidates = {}
+ evaluated_candidates = defaultdict(dict)
+
+ # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
+ for predictor in module.predictors():
+ basic_instruction = None
+ basic_prefix = None
+ *_, last_key = self._get_signature(predictor).fields.keys()
+ basic_instruction = self._get_signature(predictor).instructions
+ basic_prefix = self._get_signature(predictor).fields[last_key].json_schema_extra['prefix']
+ if self.prompt_model:
+ with dspy.settings.context(lm=self.prompt_model):
+ instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+ else:
+ instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+ # Add in our initial prompt as a candidate as well
+ instruct.completions.proposed_instruction.append(basic_instruction)
+ instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)
+ candidates[id(predictor)] = instruct.completions
+ evaluated_candidates[id(predictor)] = {}
+
+ if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
+
+ latest_candidates = candidates
+ all_candidates = candidates
+
+ module_clone = module.deepcopy()
+
+ # For each iteration in depth...
+ for d in range(self.depth): # TODO: fix this so that we eval the new batch of predictors with the new best followoing predictors
+ if self.verbose: print(f"Starting iteration {d}/{self.depth}.")
+
+ latest_scores = []
+
+ # Go through our module's predictors
+ for p_i, (p_old, p_new) in enumerate(zip(module.predictors(), module_clone.predictors())):
+ candidates_ = latest_candidates[id(p_old)] # Use the most recently generated candidates for evaluation
+ if len(module.predictors()) > 1:
+ candidates_ = all_candidates[id(p_old)] # Unless our program has multiple predictors, in which case we need to reevaluate all prompts with the new prompt(s) for the other predictor(s)
+
+ # For each candidate
+ for c_i, c in enumerate(candidates_):
+ # Get the candidate instruction and prefix
+ instruction, prefix = c.proposed_instruction.strip('"').strip(), c.proposed_prefix_for_output_field.strip('"').strip()
+
+ # Set this new module with our instruction / prefix
+ *_, last_key = self._get_signature(p_new).fields.keys()
+ updated_signature = self._get_signature(p_new) \
+ .with_instructions(instruction) \
+ .with_updated_fields(last_key, prefix=prefix)
+ self._set_signature(p_new, updated_signature)
+
+ # Score the instruction / prefix
+ if self.verbose: print("----------------")
+ for i,predictor in enumerate(module_clone.predictors()):
+ if self.verbose: print(f"Predictor {i}")
+ self._print_signature(predictor)
+ if self.verbose: print(f"At Depth {d}/{self.depth}, Evaluating Prompt Candidate #{c_i}/{len(candidates_)} for Predictor {p_i} of {len(module.predictors())}.")
+ score = evaluate(module_clone, devset=devset, **eval_kwargs)
+ if self.verbose and self.prompt_model: print(f"prompt_model.inspect_history(n=1) {self.prompt_model.inspect_history(n=1)}")
+ total_calls += 1
+ if self.verbose: print("----------------")
+
+ replace_entry = True
+ if self.verbose: print(f"(instruction, prefix) {(instruction, prefix)}")
+ # if verbose: print(f"evaluated_candidates[id(p_old)] {evaluated_candidates[id(p_old)]}")
+ if ((instruction, prefix) in evaluated_candidates[id(p_old)]):
+ # if verbose: print(f"if evaluated_candidates[id(p_old)][(instruction, prefix)] {evaluated_candidates[id(p_old)][(instruction, prefix)]}")
+ if evaluated_candidates[id(p_old)][(instruction, prefix)]["score"] >= score:
+ replace_entry = False
+
+ if replace_entry:
+ # Add it to our evaluated candidates list
+ evaluated_candidates[id(p_old)][(instruction, prefix)] = {
+ "score": score,
+ "program": module_clone.deepcopy(),
+ "instruction": instruction,
+ "prefix": prefix,
+ "depth": d,
+ }
+
+ if (len(candidates_)-self.breadth <= c_i):
+ latest_scores.append(score)
+
+ if self.track_stats:
+ results_latest[id(p_old)]["depth"].append(d)
+ results_latest[id(p_old)]["max"].append(max(latest_scores))
+ results_latest[id(p_old)]["average"].append(sum(latest_scores)/len(latest_scores))
+ results_latest[id(p_old)]["min"].append(min(latest_scores))
+ results_latest[id(p_old)]["std"].append(np.std(latest_scores))
+
+ # Now that we've evaluated the candidates, set this predictor to the best performing version
+ # to ensure the next round of scores reflect the best possible version
+ best_candidate = max(evaluated_candidates[id(p_old)].values(), key=lambda candidate: candidate['score'])
+ *_, last_key = self._get_signature(p_old).fields.keys()
+ updated_signature = self._get_signature(p_new) \
+ .with_instructions(best_candidate["instruction"]) \
+ .with_updated_fields(last_key, prefix=best_candidate["prefix"])
+ self._set_signature(p_new, updated_signature)
+ if self.verbose: print(f"Updating Predictor {id(p_old)} to:\ni: {best_candidate['instruction']}\np: {best_candidate['prefix']}")
+ if self.verbose: print("Full predictor with update: ")
+ for i,predictor in enumerate(module_clone.predictors()):
+ if self.verbose: print(f"Predictor {i}")
+ self._print_signature(predictor)
+
+ if d == self.depth-1:
+ break
+
+
+ new_candidates = {}
+ for p_base in module.predictors():
+ # Build Few-Shot Example of Optimized Prompts
+ attempts = []
+ shortest_len = self.breadth
+ shortest_len = min(len(evaluated_candidates[id(p_base)]),shortest_len)
+ best_predictors = list(evaluated_candidates[id(p_base)].values())
+
+ # best_predictors = evaluated_candidates[id(p_base)].values()[:]
+ best_predictors.sort(key=lambda x: x['score'], reverse=True)
+
+ if self.track_stats:
+ scores = [x['score'] for x in best_predictors][:10]
+ results_best[id(p_base)]["depth"].append(d)
+ results_best[id(p_base)]["max"].append(max(scores))
+ results_best[id(p_base)]["average"].append(sum(scores)/len(scores))
+ results_best[id(p_base)]["min"].append(min(scores))
+ results_best[id(p_base)]["std"].append(np.std(scores))
+
+ for i in range(shortest_len-1,-1,-1):
+ # breakpoint()
+ attempts.append(f'Instruction #{shortest_len-i}: {best_predictors[i]["instruction"]}')
+ attempts.append(f'Prefix #{shortest_len-i}: {best_predictors[i]["prefix"]}')
+ attempts.append(f'Resulting Score #{shortest_len-i}: {best_predictors[i]["score"]}')
+
+ # Generate next batch of potential prompts to optimize, with previous attempts as input
+ if self.prompt_model:
+ with dspy.settings.context(lm=self.prompt_model):
+ instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts)
+ else:
+ instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts)
+
+ if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
+ # Get candidates for each predictor
+ new_candidates[id(p_base)] = instr.completions
+ all_candidates[id(p_base)].proposed_instruction.extend(instr.completions.proposed_instruction)
+ all_candidates[id(p_base)].proposed_prefix_for_output_field.extend(instr.completions.proposed_prefix_for_output_field)
+
+ if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
+ latest_candidates = new_candidates
+
+ candidates = []
+ for predictor in module.predictors():
+ candidates.extend(list(evaluated_candidates[id(predictor)].values()))
+
+ if self.track_stats:
+ best_predictors = list(evaluated_candidates[id(predictor)].values())
+ best_predictors.sort(key=lambda x: x['score'], reverse=True)
+
+ scores = [x['score'] for x in best_predictors][:10]
+ results_best[id(predictor)]["depth"].append(d)
+ results_best[id(predictor)]["max"].append(max(scores))
+ results_best[id(predictor)]["average"].append(sum(scores)/len(scores))
+ results_best[id(predictor)]["min"].append(min(scores))
+ results_best[id(predictor)]["std"].append(np.std(scores))
+
+ # if verbose: print(f"candidates: {candidates}")
+ candidates.sort(key=lambda x: x['score'], reverse=True)
+
+ candidates = self._drop_duplicates(candidates)
+
+ best_program = candidates[0]["program"]
+ best_program.candidate_programs = candidates
+ best_program.total_calls = total_calls
+ if self.track_stats:
+ best_program.results_best = results_best
+ best_program.results_latest = results_latest
+
+ return best_program
\ No newline at end of file
diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.py
new file mode 100644
index 000000000..5dfd691ca
--- /dev/null
+++ b/dspy/teleprompt/mipro_optimizer.py
@@ -0,0 +1,499 @@
+import math
+import random
+from collections import defaultdict
+import textwrap
+
+import optuna
+
+import dsp
+import dspy
+from dspy.evaluate.evaluate import Evaluate
+from dspy.signatures import Signature
+from dspy.signatures.signature import signature_to_template
+from dspy.teleprompt import BootstrapFewShot
+from dspy.teleprompt.teleprompt import Teleprompter
+import sys
+import warnings
+
+
+"""
+USAGE SUGGESTIONS:
+
+The following code can be used to compile a optimized signature teleprompter using the MIPROOptimizer, and evaluate it on an end task:
+
+from dspy.teleprompt import MIPROOptimizer
+
+teleprompter = MIPROOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=10, init_temperature=1.0)
+kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
+compiled_prompt_opt = teleprompter.compile(program, devset=devset[:DEV_NUM], trials_num=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
+eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
+
+Note that this teleprompter takes in the following parameters:
+
+* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
+* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (ie. dspy.settings.configure(lm=task_model)).
+* metric: The task metric used for optimization.
+* n: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.
+* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.
+* verbose: Tells the method whether or not to print intermediate steps.
+* track_stats: Tells the method whether or not to track statistics about the optimization process.
+ If True, the method will track a dictionary with a key corresponding to the trial number,
+ and a value containing a dict with the following keys:
+ * program: the program being evaluated at a given trial
+ * score: the last average evaluated score for the program
+ * pruned: whether or not this program was pruned
+ This information will be returned as attributes of the best program.
+"""
+
+class BasicGenerateInstruction(Signature):
+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
+
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class BasicGenerateInstructionWithDataObservations(Signature):
+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
+
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ observations = dspy.InputField(desc="Observations about the dataset and task")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class BasicGenerateInstructionWithExamples(dspy.Signature):
+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.
+
+Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative.""")
+ # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):
+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.
+
+Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")
+ observations = dspy.InputField(desc="Observations about the dataset and task")
+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
+
+class ObservationSummarizer(dspy.Signature):
+ ("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")
+ observations = dspy.InputField(desc="Observations I have made about my dataset")
+ summary = dspy.OutputField(desc="Two to Three sentence summary of only the most significant highlights of my observations")
+
+class DatasetDescriptor(dspy.Signature):
+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
+
+ examples = dspy.InputField(desc="Sample data points from the dataset")
+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")
+
+class DatasetDescriptorWithPriorObservations(dspy.Signature):
+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """
+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
+
+ examples = dspy.InputField(desc="Sample data points from the dataset")
+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")
+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")
+
+class MIPRO(Teleprompter):
+ def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=True, view_data_batch_size=10):
+ self.n = n
+ self.metric = metric
+ self.init_temperature = init_temperature
+ self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm
+ self.task_model = task_model if task_model is not None else dspy.settings.lm
+ self.verbose = verbose
+ self.track_stats = track_stats
+ self.teacher_settings = teacher_settings
+ self.view_data_batch_size = view_data_batch_size
+
+ def _print_full_program(self, program):
+ for i,predictor in enumerate(program.predictors()):
+ if self.verbose: print(f"Predictor {i}")
+ # if self.verbose: print(f"i: {self._get_signature(predictor).instructions}")
+ # if self.verbose: print(f"p: {self._get_signature(predictor).fields[-1].name}")
+ if self.verbose: print(f"i: {self._get_signature(predictor).instructions}")
+ *_, last_field = self._get_signature(predictor).fields.values()
+ if self.verbose: print(f"p: {last_field.json_schema_extra['prefix']}")
+ if self.verbose: print("\n")
+
+ def _print_model_history(self, model, n=1):
+ if self.verbose: print(f"Model ({model}) History:")
+ model.inspect_history(n=n)
+
+ def _observe_data(self, trainset, max_iterations=10):
+ upper_lim = min(len(trainset), self.view_data_batch_size)
+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))
+ observations = observation["observations"]
+
+ skips = 0
+ iterations = 0
+ for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):
+ upper_lim = min(len(trainset), b+self.view_data_batch_size)
+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=(trainset[b:upper_lim].__repr__()))
+ iterations += 1
+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":
+ skips += 1
+ if skips >= 5:
+ break
+ continue
+ if iterations >= max_iterations:
+ break
+ observations += output["observations"]
+
+ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
+
+ return summary.summary
+
+ def _create_example_string(self, fields, example):
+
+ # Building the output string
+ output = []
+ for field in fields:
+ name = field.name
+ separator = field.separator
+ input_variable = field.input_variable
+
+ # Determine the value from input_data or prediction_data
+ value = example.get(input_variable)
+
+ # Construct the string for the current field
+ field_str = f"{name}{separator}{value}"
+ output.append(field_str)
+
+ # Joining all the field strings
+ return '\n'.join(output)
+
+ def _get_signature(self, predictor):
+ if (hasattr(predictor, 'extended_signature')):
+ return predictor.extended_signature
+ elif (hasattr(predictor, 'signature')):
+ return predictor.signature
+
+ def _set_signature(self, predictor, updated_signature):
+ if (hasattr(predictor, 'extended_signature')):
+ predictor.extended_signature = updated_signature
+ elif (hasattr(predictor, 'signature')):
+ predictor.signature = updated_signature
+
+ def _generate_first_N_candidates(self, module, N, view_data, view_examples, demo_candidates, devset):
+ candidates = {}
+ evaluated_candidates = defaultdict(dict)
+
+ if view_data:
+ # Create data observations
+ self.observations = None
+ with dspy.settings.context(lm=self.prompt_model):
+ self.observations = self._observe_data(devset).replace("Observations:","").replace("Summary:","")
+
+ if view_examples:
+ example_sets = {}
+ for predictor in module.predictors():
+ # Get all augmented examples
+ example_set = {}
+ all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples
+ for example_set_i, set_of_examples in enumerate(all_sets_of_examples):
+ if example_set_i != 0: # Skip the no examples case
+ for example in set_of_examples: # Get each individual example in the set
+ if "augmented" in example.keys():
+ if example["augmented"]:
+ if example_set_i not in example_set:
+ example_set[example_set_i] = []
+ fields_to_use = signature_to_template(predictor.signature).fields
+ input_variable_names = list(self._get_signature(predictor).input_fields.keys())
+ example_string = self._create_example_string(fields_to_use, example)
+ example_set[example_set_i].append(example_string)
+ example_sets[id(predictor)] = example_set
+ else:
+ example_set[example_set_i] = []
+ example_sets[id(predictor)] = example_set
+
+ # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
+ for predictor in module.predictors():
+ basic_instruction = None
+ basic_prefix = None
+ basic_instruction = self._get_signature(predictor).instructions
+ *_, last_field = self._get_signature(predictor).fields.values()
+ basic_prefix = last_field.json_schema_extra["prefix"]
+ with dspy.settings.context(lm=self.prompt_model):
+ # Data & Examples
+ if view_data and view_examples:
+ if 1 not in example_sets[id(predictor)].keys():
+ raise ValueError("No examples found for the given predictor")
+ instruct = None
+ for i in range(1, self.n):
+ new_instruct = dspy.Predict(
+ BasicGenerateInstructionWithExamplesAndDataObservations,
+ n=1,
+ temperature=self.init_temperature,
+ )(
+ basic_instruction=basic_instruction,
+ observations=self.observations,
+ examples=example_sets[id(predictor)][i],
+ )
+ if not instruct:
+ instruct = new_instruct
+ else:
+ instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)
+ instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)
+ # Just data
+ elif view_data:
+ instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)
+ # Just examples
+ elif view_examples:
+ instruct = None
+ for i in range(1,self.n): # Note: skip over the first example set which is empty
+ new_instruct = dspy.Predict(
+ BasicGenerateInstructionWithExamples,
+ n=1,
+ temperature=self.init_temperature,
+ )(
+ basic_instruction=basic_instruction,
+ examples=example_sets[id(predictor)][i],
+ )
+ if not instruct:
+ instruct = new_instruct
+ else:
+ instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)
+ instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)
+ # Neither
+ else:
+ instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
+
+ # Add in our initial prompt as a candidate as well
+ instruct.completions.proposed_instruction.insert(0, basic_instruction)
+ instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)
+ candidates[id(predictor)] = instruct.completions
+ evaluated_candidates[id(predictor)] = {}
+
+ if self.verbose: self._print_model_history(self.prompt_model)
+
+ return candidates, evaluated_candidates
+
+ def compile(self, student, *, devset, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, view_data=True, view_examples=True, requires_permission_to_run=True, trials_num=None, optuna_trials_num=None):
+ # Define ANSI escape codes for colors
+ YELLOW = '\033[93m'
+ BLUE = '\033[94m'
+ BOLD = '\033[1m'
+ ENDC = '\033[0m' # Resets the color to default
+
+ # Check if both trials_num and optuna_trials_num are None
+ if trials_num is None and optuna_trials_num is None:
+ raise ValueError(f"{YELLOW}{BOLD}You must specify the number of trials using the 'trials_num' parameter.{ENDC}")
+
+ # Check if the deprecated parameter is used
+ if optuna_trials_num is not None:
+ print("in it!")
+ # Issue a deprecation warning
+ warnings.warn(
+ "`trials_num` is deprecated and will be removed in a future version. "
+ "Use `trials_num` instead.",
+ DeprecationWarning
+ )
+ # Use trials_num as a fallback if trials_num is not provided
+ if trials_num is None:
+ trials_num = optuna_trials_num
+
+ random.seed(seed)
+
+ estimated_task_model_calls_wo_module_calls = len(devset) * trials_num # M * T * P
+ estimated_prompt_model_calls = 10 + self.n * len(student.predictors()) # num data summary calls + N * P
+
+
+
+ user_message = textwrap.dedent(f"""\
+ {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}
+
+ Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:
+
+ {YELLOW}- Task Model: {BLUE}{BOLD}{len(devset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{trials_num}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}
+ {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}
+
+ {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}
+
+ {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token)
+ + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC}
+
+ For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task
+ and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:
+
+ {YELLOW}- Reducing the number of trials (`trials_num`), the size of the trainset, or the number of LM calls in your program.{ENDC}
+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}
+
+ To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
+
+ If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`.{ENDC}
+
+ {YELLOW}Awaiting your input...{ENDC}
+ """)
+
+ print(user_message)
+
+ sys.stdout.flush() # Flush the output buffer to force the message to print
+
+
+ if requires_permission_to_run:
+ user_input = input("Do you wish to continue? (y/n): ").strip().lower()
+ if user_input != 'y':
+ print("Compilation aborted by the user.")
+ else:
+ # Set up program and evaluation function
+ module = student.deepcopy()
+ evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
+
+ # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt
+ if max_bootstrapped_demos==0 and max_labeled_demos==0: #TODO: address case when max_bootstrapped alone is 0
+ max_bootstrapped_demos_for_candidate_gen = 1
+ max_labeled_demos_for_candidate_gen = 1 #TODO: this might only need to be 0
+ else:
+ max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos
+ max_labeled_demos_for_candidate_gen = max_labeled_demos
+
+ # Generate N few shot example sets
+ demo_candidates = {}
+ for i in range(self.n):
+ if i == 0: # Story empty set of demos as default for index 0
+ for module_p in module.predictors():
+ if id(module_p) not in demo_candidates.keys():
+ demo_candidates[id(module_p)] = []
+ demo_candidates[id(module_p)].append([])
+ else:
+ if self.verbose: print(f"Creating basic bootstrap: {i}/{self.n-1}")
+
+ # Create a new basic bootstrap few - shot program .
+ rng = random.Random(i)
+ shuffled_devset = devset[:] # Create a copy of devset
+ rng.shuffle(shuffled_devset) # Shuffle the copy
+ tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen, teacher_settings=self.teacher_settings)
+ candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_devset)
+
+ # Store the candidate demos
+ for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):
+ if id(module_p) not in demo_candidates.keys():
+ demo_candidates[id(module_p)] = []
+ demo_candidates[id(module_p)].append(candidate_p.demos)
+
+ # Generate N candidate prompts
+ instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, view_data, view_examples, demo_candidates, devset)
+
+ # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples
+ if max_bootstrapped_demos==0 and max_labeled_demos==0:
+ demo_candidates = None
+
+ # Initialize variables to store the best program and its score
+ best_score = float('-inf')
+ best_program = None
+ trial_num = 0
+
+ trial_logs = {}
+
+ # Define our trial objective
+ def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, devset):
+ def objective(trial):
+ nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables
+ candidate_program = baseline_program.deepcopy()
+
+ # Suggest the instruction to use for our predictor
+ print(f"Starting trial #{trial_num}")
+ trial_logs[trial_num] = {}
+
+ for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):
+
+ # Get instruction candidates for our given predictor
+ p_instruction_candidates = instruction_candidates[id(p_old)]
+ if demo_candidates: p_demo_candidates = demo_candidates[id(p_old)]
+
+ # Suggest the index of the instruction candidate to use in our trial
+ instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))
+ if demo_candidates: demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))
+ trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx
+ if demo_candidates: trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx
+
+ # Get the selected instruction candidate
+ selected_candidate = p_instruction_candidates[instruction_idx]
+ selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()
+ selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()
+
+ # Use this candidates in our program
+ *_, last_field = self._get_signature(p_new).fields.keys()
+ updated_signature = self._get_signature(p_new).with_instructions(selected_instruction).with_updated_fields(last_field, prefix=selected_prefix)
+ self._set_signature(p_new, updated_signature)
+
+ # Get the selected demos
+ if demo_candidates: selected_demos = p_demo_candidates[demos_idx]
+
+ # Use these demos in our program
+ if demo_candidates: p_new.demos = selected_demos
+
+ # breakpoint()
+
+ if self.verbose: print("Evaling the following program:")
+ # breakpoint()
+ if self.verbose: self._print_full_program(candidate_program)
+ trial_logs[trial_num]["program"] = candidate_program
+
+ # Evaluate with the new prompts
+ total_score = 0
+ batch_size = 100
+ num_batches = math.ceil(len(devset) / batch_size)
+
+ for i in range(num_batches):
+ start_index = i * batch_size
+ end_index = min((i + 1) * batch_size, len(devset))
+ split_dev = devset[start_index:end_index]
+ split_score = evaluate(candidate_program, devset=split_dev, display_table=0)
+ if self.verbose: print(f"{i}st split score: {split_score}")
+
+ total_score += split_score * len(split_dev)
+ curr_weighted_avg_score = total_score / min((i+1)*100,len(devset))
+ if self.verbose: print(f"curr average score: {curr_weighted_avg_score}")
+
+ trial.report(curr_weighted_avg_score, i)
+
+ # Handle pruning based on the intermediate value.
+ if trial.should_prune():
+ print(f"Trial pruned.")
+ trial_logs[trial_num]["score"] = curr_weighted_avg_score
+ trial_logs[trial_num]["pruned"] = True
+ trial_num += 1
+ raise optuna.TrialPruned()
+
+ if self.verbose: print(f"Fully evaled score: {curr_weighted_avg_score}")
+ if self.verbose: self._print_model_history(self.task_model, n=1)
+ # breakpoint()
+ score = curr_weighted_avg_score
+
+ trial_logs[trial_num]["score"] = curr_weighted_avg_score
+ trial_logs[trial_num]["pruned"] = False
+
+ # Update the best program if the current score is better
+ if score > best_score:
+ best_score = score
+ best_program = candidate_program.deepcopy()
+
+ trial_num += 1
+
+ return score
+
+ return objective
+
+ # Run the trial
+ objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, devset)
+ sampler = optuna.samplers.TPESampler(seed=seed)
+ study = optuna.create_study(direction="maximize", sampler=sampler)
+ score = study.optimize(objective_function, n_trials=trials_num)
+
+ if best_program is not None and self.track_stats:
+ best_program.trial_logs = trial_logs
+
+ print(f"Returning {best_program} from continue_program")
+ return best_program
\ No newline at end of file
diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.py
index e760ebf45..04da45a9c 100644
--- a/dspy/teleprompt/signature_opt.py
+++ b/dspy/teleprompt/signature_opt.py
@@ -1,12 +1,10 @@
-from collections import defaultdict
-
-import dsp
-import dspy
-from dspy.evaluate.evaluate import Evaluate
-from dspy.signatures import Signature
-from dspy.teleprompt.teleprompt import Teleprompter
-
+from .copro_optimizer import COPRO
"""
+===============================================================
+DEPRECATED!!!
+PLEASE USE COPRO INSTEAD.
+===============================================================
+
USAGE SUGGESTIONS:
The following code can be used to compile a optimized signature teleprompter, and evaluate it on an end task:
@@ -31,284 +29,7 @@ Note that this teleprompter takes in the following parameters:
* total_calls: The total number of calls to the task metric.
These statistics will be returned as attributes of the best program.
"""
-class BasicGenerateInstruction(Signature):
- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class GenerateInstructionGivenAttempts(dspy.Signature):
- """You are an instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality.
-Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative."""
-
- attempted_instructions = dspy.InputField(format=dsp.passages2text)
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class SignatureOptimizer(Teleprompter):
+class SignatureOptimizer(COPRO):
def __init__(self, prompt_model=None, metric=None, breadth=10, depth=3, init_temperature=1.4, verbose=False, track_stats=False):
- if breadth <= 1:
- raise ValueError("Breadth must be greater than 1")
- self.metric = metric
- self.breadth = breadth
- self.depth = depth
- self.init_temperature = init_temperature
- self.prompt_model = prompt_model
- self.verbose = verbose
- self.track_stats = track_stats
-
- def _check_candidates_equal(self, candidate1, candidate2):
- for p1, p2 in zip(candidate1["program"].predictors(), candidate2["program"].predictors()):
- if p1.extended_signature.instructions != p2.extended_signature.instructions:
- return False
- *_, p1_last_field = p1.extended_signature.fields.values()
- *_, p2_last_field = p2.extended_signature.fields.values()
- if p1_last_field != p2_last_field:
- return False
- return True
-
- def _drop_duplicates(self, candidates):
- final_candidates = []
- last_batch = []
- last_batch_score = -1
- for c in candidates:
- repeat = False
- if c['score'] == last_batch_score:
- for c2 in last_batch:
- if (self._check_candidates_equal(c, c2)):
- repeat = True
- break
- if not repeat:
- last_batch.append(c)
- else:
- last_batch = [c]
- last_batch_score = c['score']
- if not repeat:
- final_candidates.append(c)
- return final_candidates
-
- def _print_signature(self, predictor):
- if self.verbose:
- if (hasattr(predictor, 'extended_signature')):
- signature = predictor.extended_signature
- else:
- signature = predictor.extended_signature1
- print(f"i: {signature.instructions}")
- print(f"p: {list(signature.fields.values())[-1].json_schema_extra['prefix']}")
- print()
-
-
- def compile(self, student, *, devset, eval_kwargs):
- """student is a program that needs to be optimized, note that it may be zero-shot or already pre-optimized for demos != []"""
- module = student.deepcopy()
- evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
- total_calls = 0
- results_best = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}
- results_latest = {id(p):{"depth": [], "max": [], "average": [], "min":[], "std": []} for p in module.predictors()}
-
- if self.track_stats:
- import numpy as np
-
-
- candidates = {}
- evaluated_candidates = defaultdict(dict)
-
- # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
- for predictor in module.predictors():
- basic_instruction = None
- basic_prefix = None
- *_, last_key = predictor.extended_signature.fields.keys()
- if (hasattr(predictor, 'extended_signature')):
- basic_instruction = predictor.extended_signature.instructions
- basic_prefix = predictor.extended_signature.fields[last_key].json_schema_extra['prefix']
- else:
- basic_instruction = predictor.extended_signature1.instructions
- basic_prefix = predictor.extended_signature1.fields[last_key].json_schema_extra['prefix']
- if self.prompt_model:
- with dspy.settings.context(lm=self.prompt_model):
- instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
- else:
- instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
- # Add in our initial prompt as a candidate as well
- instruct.completions.proposed_instruction.append(basic_instruction)
- instruct.completions.proposed_prefix_for_output_field.append(basic_prefix)
- candidates[id(predictor)] = instruct.completions
- evaluated_candidates[id(predictor)] = {}
-
- if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
-
- latest_candidates = candidates
- all_candidates = candidates
-
- module_clone = module.deepcopy()
-
- # For each iteration in depth...
- for d in range(self.depth): # TODO: fix this so that we eval the new batch of predictors with the new best followoing predictors
- if self.verbose: print(f"Starting iteration {d}/{self.depth}.")
-
- latest_scores = []
-
- # Go through our module's predictors
- for p_i, (p_old, p_new) in enumerate(zip(module.predictors(), module_clone.predictors())):
- candidates_ = latest_candidates[id(p_old)] # Use the most recently generated candidates for evaluation
- if len(module.predictors()) > 1:
- candidates_ = all_candidates[id(p_old)] # Unless our program has multiple predictors, in which case we need to reevaluate all prompts with the new prompt(s) for the other predictor(s)
-
- # For each candidate
- for c_i, c in enumerate(candidates_):
- # Get the candidate instruction and prefix
- instruction, prefix = c.proposed_instruction.strip('"').strip(), c.proposed_prefix_for_output_field.strip('"').strip()
-
- # Set this new module with our instruction / prefix
- if (hasattr(p_new, 'extended_signature')):
- *_, last_key = p_new.extended_signature.fields.keys()
- p_new.extended_signature = p_new.extended_signature \
- .with_instructions(instruction) \
- .with_updated_fields(last_key, prefix=prefix)
- else:
- *_, last_key = p_new.extended_signature1.fields.keys()
- p_new.extended_signature1 = p_new.extended_signature1 \
- .with_instructions(instruction) \
- .with_updated_fields(last_key, prefix=prefix)
- *_, last_key = p_new.extended_signature2.fields.keys()
- p_new.extended_signature2 = p_new.extended_signature2 \
- .with_instructions(instruction) \
- .with_updated_fields(last_key, prefix=prefix)
-
- # Score the instruction / prefix
- if self.verbose: print("----------------")
- for i,predictor in enumerate(module_clone.predictors()):
- if self.verbose: print(f"Predictor {i}")
- self._print_signature(predictor)
- if self.verbose: print(f"At Depth {d}/{self.depth}, Evaluating Prompt Candidate #{c_i}/{len(candidates_)} for Predictor {p_i} of {len(module.predictors())}.")
- score = evaluate(module_clone, devset=devset, **eval_kwargs)
- if self.verbose and self.prompt_model: print(f"prompt_model.inspect_history(n=1) {self.prompt_model.inspect_history(n=1)}")
- total_calls += 1
- if self.verbose: print("----------------")
-
- replace_entry = True
- if self.verbose: print(f"(instruction, prefix) {(instruction, prefix)}")
- # if verbose: print(f"evaluated_candidates[id(p_old)] {evaluated_candidates[id(p_old)]}")
- if ((instruction, prefix) in evaluated_candidates[id(p_old)]):
- # if verbose: print(f"if evaluated_candidates[id(p_old)][(instruction, prefix)] {evaluated_candidates[id(p_old)][(instruction, prefix)]}")
- if evaluated_candidates[id(p_old)][(instruction, prefix)]["score"] >= score:
- replace_entry = False
-
- if replace_entry:
- # Add it to our evaluated candidates list
- evaluated_candidates[id(p_old)][(instruction, prefix)] = {
- "score": score,
- "program": module_clone.deepcopy(),
- "instruction": instruction,
- "prefix": prefix,
- "depth": d,
- }
-
- if (len(candidates_)-self.breadth <= c_i):
- latest_scores.append(score)
-
- if self.track_stats:
- results_latest[id(p_old)]["depth"].append(d)
- results_latest[id(p_old)]["max"].append(max(latest_scores))
- results_latest[id(p_old)]["average"].append(sum(latest_scores)/len(latest_scores))
- results_latest[id(p_old)]["min"].append(min(latest_scores))
- results_latest[id(p_old)]["std"].append(np.std(latest_scores))
-
- # Now that we've evaluated the candidates, set this predictor to the best performing version
- # to ensure the next round of scores reflect the best possible version
- best_candidate = max(evaluated_candidates[id(p_old)].values(), key=lambda candidate: candidate['score'])
- if (hasattr(p_new, 'extended_signature')):
- *_, last_key = p_old.extended_signature.fields.keys()
- p_new.extended_signature = p_new.extended_signature \
- .with_instructions(best_candidate["instruction"]) \
- .with_updated_fields(last_key, prefix=best_candidate["prefix"])
- else:
- *_, last_key1 = p_old.extended_signature1.fields.keys()
- p_new.extended_signature1 = p_new.extended_signature \
- .with_instructions(best_candidate["instruction"]) \
- .with_updated_fields(last_key1, prefix=best_candidate["prefix"])
- *_, last_key2 = p_old.extended_signature2.fields.keys()
- p_new.extended_signature2 = p_new.extended_signature \
- .with_instructions(best_candidate["instruction"]) \
- .with_updated_fields(last_key2, prefix=best_candidate["prefix"])
- if self.verbose: print(f"Updating Predictor {id(p_old)} to:\ni: {best_candidate['instruction']}\np: {best_candidate['prefix']}")
- if self.verbose: print("Full predictor with update: ")
- for i,predictor in enumerate(module_clone.predictors()):
- if self.verbose: print(f"Predictor {i}")
- self._print_signature(predictor)
-
- if d == self.depth-1:
- break
-
-
- new_candidates = {}
- for p_base in module.predictors():
- # Build Few-Shot Example of Optimized Prompts
- attempts = []
- shortest_len = self.breadth
- shortest_len = min(len(evaluated_candidates[id(p_base)]),shortest_len)
- best_predictors = list(evaluated_candidates[id(p_base)].values())
-
- # best_predictors = evaluated_candidates[id(p_base)].values()[:]
- best_predictors.sort(key=lambda x: x['score'], reverse=True)
-
- if self.track_stats:
- scores = [x['score'] for x in best_predictors][:10]
- results_best[id(p_base)]["depth"].append(d)
- results_best[id(p_base)]["max"].append(max(scores))
- results_best[id(p_base)]["average"].append(sum(scores)/len(scores))
- results_best[id(p_base)]["min"].append(min(scores))
- results_best[id(p_base)]["std"].append(np.std(scores))
-
- for i in range(shortest_len-1,-1,-1):
- # breakpoint()
- attempts.append(f'Instruction #{shortest_len-i}: {best_predictors[i]["instruction"]}')
- attempts.append(f'Prefix #{shortest_len-i}: {best_predictors[i]["prefix"]}')
- attempts.append(f'Resulting Score #{shortest_len-i}: {best_predictors[i]["score"]}')
-
- # Generate next batch of potential prompts to optimize, with previous attempts as input
- if self.prompt_model:
- with dspy.settings.context(lm=self.prompt_model):
- instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts)
- else:
- instr = dspy.Predict(GenerateInstructionGivenAttempts, n=self.breadth, temperature=self.init_temperature)(attempted_instructions=attempts)
-
- if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
- # Get candidates for each predictor
- new_candidates[id(p_base)] = instr.completions
- all_candidates[id(p_base)].proposed_instruction.extend(instr.completions.proposed_instruction)
- all_candidates[id(p_base)].proposed_prefix_for_output_field.extend(instr.completions.proposed_prefix_for_output_field)
-
- if self.verbose and self.prompt_model: print(f"{self.prompt_model.inspect_history(n=1)}")
- latest_candidates = new_candidates
-
- candidates = []
- for predictor in module.predictors():
- candidates.extend(list(evaluated_candidates[id(predictor)].values()))
-
- if self.track_stats:
- best_predictors = list(evaluated_candidates[id(predictor)].values())
- best_predictors.sort(key=lambda x: x['score'], reverse=True)
-
- scores = [x['score'] for x in best_predictors][:10]
- results_best[id(predictor)]["depth"].append(d)
- results_best[id(predictor)]["max"].append(max(scores))
- results_best[id(predictor)]["average"].append(sum(scores)/len(scores))
- results_best[id(predictor)]["min"].append(min(scores))
- results_best[id(predictor)]["std"].append(np.std(scores))
-
- # if verbose: print(f"candidates: {candidates}")
- candidates.sort(key=lambda x: x['score'], reverse=True)
-
- candidates = self._drop_duplicates(candidates)
-
- best_program = candidates[0]["program"]
- best_program.candidate_programs = candidates
- best_program.total_calls = total_calls
- if self.track_stats:
- best_program.results_best = results_best
- best_program.results_latest = results_latest
-
- return best_program
\ No newline at end of file
+ super().__init__(prompt_model, metric, breadth, depth, init_temperature, verbose, track_stats)
\ No newline at end of file
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.py
index e316462c9..a0cdd8458 100644
--- a/dspy/teleprompt/signature_opt_bayesian.py
+++ b/dspy/teleprompt/signature_opt_bayesian.py
@@ -1,18 +1,11 @@
-import math
-import random
-from collections import defaultdict
-
-import optuna
-
-import dsp
-import dspy
-from dspy.evaluate.evaluate import Evaluate
-from dspy.signatures import Signature
-from dspy.signatures.signature import signature_to_template
-from dspy.teleprompt import BootstrapFewShot
-from dspy.teleprompt.teleprompt import Teleprompter
+from dspy.teleprompt.mipro_optimizer import MIPRO
"""
+===============================================================
+DEPRECATED!!!
+PLEASE USE MIPRO INSTEAD.
+===============================================================
+
USAGE SUGGESTIONS:
The following code can be used to compile a optimized signature teleprompter using the BayesianSignatureOptimizer, and evaluate it on an end task:
@@ -40,373 +33,9 @@ Note that this teleprompter takes in the following parameters:
* pruned: whether or not this program was pruned
This information will be returned as attributes of the best program.
"""
-class BasicGenerateInstruction(Signature):
- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class BasicGenerateInstructionWithDataObservations(Signature):
- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""
-
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
- observations = dspy.InputField(desc="Observations about the dataset and task")
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class BasicGenerateInstructionWithExamples(dspy.Signature):
- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.
-
-Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative.""")
- # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
- # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):
- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs.
-
-Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
- observations = dspy.InputField(desc="Observations about the dataset and task")
- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
-
-class ObservationSummarizer(dspy.Signature):
- ("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")
- observations = dspy.InputField(desc="Observations I have made about my dataset")
- summary = dspy.OutputField(desc="Two to Three sentence summary of only the most significant highlights of my observations")
-
-class DatasetDescriptor(dspy.Signature):
- ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
-
- examples = dspy.InputField(desc="Sample data points from the dataset")
- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")
-
-class DatasetDescriptorWithPriorObservations(dspy.Signature):
- ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """
- """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """
- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """
- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")
-
- examples = dspy.InputField(desc="Sample data points from the dataset")
- prior_observations = dspy.InputField(desc="Some prior observations I made about the data")
- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")
-
-class BayesianSignatureOptimizer(Teleprompter):
- def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False, view_data_batch_size=10):
- self.n = n
- self.metric = metric
- self.init_temperature = init_temperature
- self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm
- self.task_model = task_model if task_model is not None else dspy.settings.lm
- self.verbose = verbose
- self.track_stats = track_stats
- self.teacher_settings = teacher_settings
- self.view_data_batch_size = view_data_batch_size
-
- def _print_full_program(self, program):
- for i,predictor in enumerate(program.predictors()):
- if self.verbose: print(f"Predictor {i}")
- if (hasattr(predictor, 'extended_signature')):
- if self.verbose: print(f"i: {predictor.extended_signature.instructions}")
- *_, last_field = predictor.extended_signature.fields.values()
- if self.verbose: print(f"p: {last_field.json_schema_extra['prefix']}")
- else:
- if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")
- *_, last_field = predictor.extended_signature1.fields.values()
- if self.verbose: print(f"p: {last_field.json_schema_extra['prefix']}")
- if self.verbose: print("\n")
-
- def _print_model_history(self, model, n=1):
- if self.verbose: print(f"Model ({model}) History:")
- model.inspect_history(n=n)
-
- def _observe_data(self, trainset):
- upper_lim = min(len(trainset), self.view_data_batch_size)
- observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))
- observations = observation["observations"]
-
- skips = 0
- for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):
- upper_lim = min(len(trainset), b+self.view_data_batch_size)
- output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=(trainset[b:upper_lim].__repr__()))
- if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":
- skips += 1
- if skips >= 5:
- break
- continue
- observations += output["observations"]
-
- summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
-
- return summary.summary
-
- def _create_example_string(self, fields, example):
-
- # Building the output string
- output = []
- for field in fields:
- name = field.name
- separator = field.separator
- input_variable = field.input_variable
-
- # Determine the value from input_data or prediction_data
- value = example.get(input_variable)
-
- # Construct the string for the current field
- field_str = f"{name}{separator}{value}"
- output.append(field_str)
-
- # Joining all the field strings
- return '\n'.join(output)
-
- def _generate_first_N_candidates(self, module, N, view_data, view_examples, demo_candidates, devset):
- candidates = {}
- evaluated_candidates = defaultdict(dict)
-
- if view_data:
- # Create data observations
- self.observations = None
- with dspy.settings.context(lm=self.prompt_model):
- self.observations = self._observe_data(devset).replace("Observations:","").replace("Summary:","")
-
- if view_examples:
- example_sets = {}
- for predictor in module.predictors():
- # Get all augmented examples
- example_set = {}
- all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples
- for example_set_i, set_of_examples in enumerate(all_sets_of_examples):
- if example_set_i != 0: # Skip the no examples case
- for example in set_of_examples: # Get each individual example in the set
- if "augmented" in example.keys():
- if example["augmented"]:
- if example_set_i not in example_set:
- example_set[example_set_i] = []
- fields_to_use = signature_to_template(predictor.signature).fields
- input_variable_names = list(predictor.signature.input_fields.keys())
- example_with_only_signature_fields = {key: value for key, value in example.items() if key in input_variable_names}
- example_string = self._create_example_string(fields_to_use, example_with_only_signature_fields)
- example_set[example_set_i].append(example_string)
- example_sets[id(predictor)] = example_set
- else:
- example_set[example_set_i] = []
- example_sets[id(predictor)] = example_set
-
- # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts
- for predictor in module.predictors():
- basic_instruction = None
- basic_prefix = None
- if (hasattr(predictor, 'extended_signature')):
- basic_instruction = predictor.extended_signature.instructions
- *_, last_field = predictor.extended_signature.fields.values()
- basic_prefix = last_field.json_schema_extra["prefix"]
- else:
- basic_instruction = predictor.extended_signature1.instructions
- *_, last_field = predictor.extended_signature1.fields.values()
- basic_prefix = last_field.json_schema_extra["prefix"]
- with dspy.settings.context(lm=self.prompt_model):
- # Data & Examples
- if view_data and view_examples:
- if 1 not in example_sets[id(predictor)].keys():
- raise ValueError("No examples found for the given predictor")
- instruct = None
- for i in range(1, self.n):
- new_instruct = dspy.Predict(
- BasicGenerateInstructionWithExamplesAndDataObservations,
- n=1,
- temperature=self.init_temperature,
- )(
- basic_instruction=basic_instruction,
- observations=self.observations,
- examples=example_sets[id(predictor)][i],
- )
- if not instruct:
- instruct = new_instruct
- else:
- instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)
- instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)
- # Just data
- elif view_data:
- instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)
- # Just examples
- elif view_examples:
- instruct = None
- for i in range(1,self.n): # Note: skip over the first example set which is empty
- new_instruct = dspy.Predict(
- BasicGenerateInstructionWithExamples,
- n=1,
- temperature=self.init_temperature,
- )(
- basic_instruction=basic_instruction,
- examples=example_sets[id(predictor)][i],
- )
- if not instruct:
- instruct = new_instruct
- else:
- instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)
- instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)
- # Neither
- else:
- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)
-
- # Add in our initial prompt as a candidate as well
- instruct.completions.proposed_instruction.insert(0, basic_instruction)
- instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)
- candidates[id(predictor)] = instruct.completions
- evaluated_candidates[id(predictor)] = {}
-
- if self.verbose: self._print_model_history(self.prompt_model)
-
- return candidates, evaluated_candidates
-
- def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, view_data=True, view_examples=True):
-
- random.seed(seed)
-
- # Set up program and evaluation function
- module = student.deepcopy()
- evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
-
- # Generate N few shot example sets
- demo_candidates = {}
- for i in range(self.n):
- if i == 0: # Story empty set of demos as default for index 0
- for module_p in module.predictors():
- if id(module_p) not in demo_candidates:
- demo_candidates[id(module_p)] = []
- demo_candidates[id(module_p)].append([])
- else:
- if self.verbose: print(f"Creating basic bootstrap: {i}/{self.n-1}")
-
- # Create a new basic bootstrap few - shot program .
- rng = random.Random(i)
- shuffled_devset = devset[:] # Create a copy of devset
- rng.shuffle(shuffled_devset) # Shuffle the copy
- tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos, max_labeled_demos=max_labeled_demos, teacher_settings=self.teacher_settings)
- candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_devset)
-
- # Store the candidate demos
- for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):
- if id(module_p) not in demo_candidates:
- demo_candidates[id(module_p)] = []
- demo_candidates[id(module_p)].append(candidate_p.demos)
-
- # Generate N candidate prompts
- instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, view_data, view_examples, demo_candidates, devset)
-
- # Initialize variables to store the best program and its score
- best_score = float('-inf')
- best_program = None
- trial_num = 0
-
- trial_logs = {}
-
- # Define our trial objective
- def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, devset):
- def objective(trial):
- nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables
- candidate_program = baseline_program.deepcopy()
-
- # Suggest the instruction to use for our predictor
- if self.verbose: print(f"Starting trial num: {trial_num}")
- trial_logs[trial_num] = {}
-
- for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):
-
- # Get instruction candidates for our given predictor
- p_instruction_candidates = instruction_candidates[id(p_old)]
- p_demo_candidates = demo_candidates[id(p_old)]
-
- # Suggest the index of the instruction candidate to use in our trial
- #instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))
- #demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))
- instruction_idx = trial.suggest_int(f"{id(p_old)}_predictor_instruction",low=0, high=len(p_instruction_candidates)-1)
- demos_idx = trial.suggest_int(f"{id(p_old)}_predictor_demos",low=0, high=len(p_demo_candidates)-1)
-
- trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx
- trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx
-
- # Get the selected instruction candidate
- selected_candidate = p_instruction_candidates[instruction_idx]
- selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()
- selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()
-
- # Use this candidates in our program
- *_, last_field = p_new.extended_signature.fields.keys()
- p_new.extended_signature = p_new.extended_signature \
- .with_instructions(selected_instruction) \
- .with_updated_fields(last_field, prefix=selected_prefix)
-
- # Get the selected demos
- selected_demos = p_demo_candidates[demos_idx]
-
- # Use these demos in our program
- p_new.demos = selected_demos
-
- if self.verbose: print("Evaling the following program:")
- self._print_full_program(candidate_program)
- trial_logs[trial_num]["program"] = candidate_program
-
- # Evaluate with the new prompts
- total_score = 0
- batch_size = 100
- num_batches = math.ceil(len(devset) / batch_size)
-
- for i in range(num_batches):
- start_index = i * batch_size
- end_index = min((i + 1) * batch_size, len(devset))
- split_dev = devset[start_index:end_index]
- split_score = evaluate(candidate_program, devset=split_dev, display_table=0)
- if self.verbose: print(f"{i}st split score: {split_score}")
-
- total_score += split_score * len(split_dev)
- curr_weighted_avg_score = total_score / min((i+1)*100,len(devset))
- if self.verbose: print(f"curr average score: {curr_weighted_avg_score}")
-
- trial.report(curr_weighted_avg_score, i)
-
- # Handle pruning based on the intermediate value.
- if trial.should_prune():
- if self.verbose: print("Optuna decided to prune!")
- trial_logs[trial_num]["score"] = curr_weighted_avg_score
- trial_logs[trial_num]["pruned"] = True
- trial_num += 1
- raise optuna.TrialPruned()
-
- if self.verbose:
- print(f"Fully evaled score: {curr_weighted_avg_score}")
- self._print_model_history(self.task_model, n=1)
- score = curr_weighted_avg_score
-
- trial_logs[trial_num]["score"] = curr_weighted_avg_score
- trial_logs[trial_num]["pruned"] = False
-
- # Update the best program if the current score is better
- if score > best_score:
- best_score = score
- best_program = candidate_program.deepcopy()
-
- trial_num += 1
-
- return score
-
- return objective
-
- # Run the trial
- objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, devset)
- sampler = optuna.samplers.TPESampler(seed=seed)
- study = optuna.create_study(direction="maximize", sampler=sampler)
- score = study.optimize(objective_function, n_trials=optuna_trials_num)
- if best_program is not None and self.track_stats:
- best_program.trial_logs = trial_logs
+class BayesianSignatureOptimizer(MIPRO):
+ def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=True, view_data_batch_size=10):
+ print(u"\u001b[31m[WARNING] BayesianSignatureOptimizer has been deprecated and replaced with MIPRO. BayesianSignatureOptimizer will be removed in a future release. \u001b[31m")
- return best_program
+ super().__init__(prompt_model, task_model, teacher_settings,n,metric,init_temperature,verbose,track_stats,view_data_batch_size)
\ No newline at end of file
diff --git a/examples/qa/hotpot/hotpotqa_optimized.ipynb b/examples/qa/hotpot/hotpotqa_optimized.ipynb
new file mode 100644
index 000000000..c87d47497
--- /dev/null
+++ b/examples/qa/hotpot/hotpotqa_optimized.ipynb
@@ -0,0 +1,566 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<img src=\"../../../docs/images/DSPy8.png\" alt=\"DSPy7 Image\" height=\"150\"/>\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Using __Multi-stage Instruction Proposal & Optimization (MIPRO)__ in DSPy"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### FAQ 🙋\n",
+ "#### 1) How does MIPRO work?\n",
+ "At a high level, the MIPRO program optimizer works by first proposing candidate fewshot example sets and instructions for each prompt in your program, and then optimizing over these fewshot example sets and instructions as hyperparameters for a specified number of trials. Each trial, the optimizer evaluates different combinations of prompts on a train set, which allows it to learn which combinations yield the best performance.\n",
+ "\n",
+ "#### 2) How much will MIPRO cost me to run?\n",
+ "Note that __this notebook__ is free to run, because all LM calls have been cached. However, when using an optimizer on your own program, here is a breakdown of the upper bound of the number of calls to the task model and prompt model respectively:\n",
+ "\n",
+ "- **Task model calls**: MIPRO makes up to __O(TxPxM)__ task model calls, where T is the number of trials, P is the number of prompts in the program, and M is the size of the train set. This is because the model is evaluating the program on the train set each trial. In practice, this should be lower given that MIPRO tunes poor trials early (ie. it may stop a trial after running on the first 100 or so examples if performance is poor).\n",
+ "\n",
+ "- **Prompt model calls**: MIPRO makes up to N*P+10 prompt model calls, where N is the number of instruction / fewshot example set candidates to generate for each prompt, and P is the number of prompts in the program. The extra 10 calls comes from generating a summary of the data in the training set, which we use in the meta prompt to create better instructions.\n",
+ "\n",
+ "#### 3) How should I configure the hyperparameters?\n",
+ "We have yet to run full hyperparameter sweeps with MIPRO, but based off of initial experimintation, we'd recommend the following:\n",
+ "- __Trial num__: Gains can be seen after about 20-30 trials. However, 100-200 trials can help with adding on additional marginal gains.\n",
+ "- __n__: This hyperparameter controls the number of candidate prompts and fewshot example sets that are generated to optimize over. With more trials and less prompts to optimize, we can set n to be higher, as we have more trials to explore different combinations of prompts. If your program has between 2-3 modules and is the `num_trials=30`, we'd recommend ~`n=10`. If n is higher (say `n=100`), then we can go higher to ~`n=15`. If you have a program with only 1 module and are keeping the program 0-shot (ie. no fewshot examples), then `num_trials` should be set to equal `n`, because each trial can explore a new instruction.\n",
+ "- __Training set size__: Between 200 and 500 training examples are recommended. Increasing the training set size can help prevent overfitting, but adds to the expense to run.\n",
+ "\n",
+ "#### 4) What should I do if I want to reduce the cost?\n",
+ "You can always update hyperparameters accordingly, such as using a smaller train set, using less trials, or using a program with less modules.\n",
+ "Alternatively, one strategy would be to optimize using a cheaper task model (ie. locally hosted Llama-2), as initial experiments have shown that prompts optimized for a smaller model also transfer to working well on a larger model.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 0] Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First, we'll __load in the cached requests__ for this tasks, so that we don't actually need to call any LMs for this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from huggingface_hub import hf_hub_download\n",
+ "import zipfile\n",
+ "import os\n",
+ "\n",
+ "# Define the repository ID on Hugging Face\n",
+ "repo_id = 'kopsahlong/test3'\n",
+ "cache_file_path = hf_hub_download(repo_id=repo_id, filename='notebook_cache_v3.zip')\n",
+ "compiled_program_file_path = hf_hub_download(repo_id=repo_id, filename='compiled_program.pickle')\n",
+ "# Unzipping the file\n",
+ "with zipfile.ZipFile(cache_file_path, 'r') as zip_ref:\n",
+ " zip_ref.extractall(\".\")\n",
+ "\n",
+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"notebook_cache\"\n",
+ "\n",
+ "\n",
+ "# Set up the cache for this notebook\n",
+ "# os.environ[\"DSP_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/caches/cache_train_500_eval_500_n_10_trials_30_hops_4\" # repo_clone_path\n",
+ "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/caches/MIPRO_notebook_cache_v2\" # repo_clone_path\n",
+ "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/notebook_cache\" # repo_clone_path\n",
+ "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/DSPy_notebook_cache/cache_copy\" "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Pydantic version: 2.6.3\n"
+ ]
+ }
+ ],
+ "source": [
+ "import pkg_resources\n",
+ "\n",
+ "# Get the version of cloudpickle\n",
+ "pydantic_v = pkg_resources.get_distribution(\"pydantic\").version\n",
+ "\n",
+ "print(f\"Pydantic version: {pydantic_v}\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# TODO: add in DSPy setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will also specify the __prompt LM model__ (in this case GPT 3.5), the __task LM model__ (Llama 13B) and the retrieval model we'll be using for our task (a HotPotQA multihop retrieval task)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os \n",
+ "import dspy\n",
+ "import openai\n",
+ "import os\n",
+ "\n",
+ "### NOTE: if you'd like to run this code without a cache, you can remove these lines to configure your OPEN AI key ###\n",
+ "# os.environ['OPENAI_API_KEY'] = \"TODO: ADD YOUR OPEN AI KEY HERE\"\n",
+ "# openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
+ "# openai.api_base = \"https://api.openai.com/v1\"\n",
+ "\n",
+ "prompt_model_name = \"gpt-3.5-turbo-1106\"\n",
+ "task_model_name = \"meta-llama/Llama-2-13b-chat-hf\"\n",
+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",
+ "\n",
+ "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=150)\n",
+ "task_model = dspy.HFClientTGI(model=task_model_name, port=[7140, 7141, 7142, 7143], max_tokens=150)\n",
+ "\n",
+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",
+ "\n",
+ "dspy.settings.configure(rm=colbertv2, lm=task_model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 1] Define Task"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here, we'll define the program that we'd like to run, which is a multihop [...] (we can say that it was loosely inspired by a certain paper). We additionally load in the data, and define how we'd like to evaluate this task."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.\n",
+ " table = cls._concat_blocks(blocks, axis=0)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.evaluate import Evaluate\n",
+ "import re \n",
+ "from dspy.datasets import HotPotQA\n",
+ "\n",
+ "class ReturnRankedDocuments(dspy.Signature):\n",
+ " \"\"\"Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\"\"\"\n",
+ " question = dspy.InputField(desc=\"The question we're trying to answer.\")\n",
+ " context = dspy.InputField(desc=\"List of potentially related passages.\")\n",
+ " ranking = dspy.OutputField(desc=\"A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.\")\n",
+ "\n",
+ "class RankingMultiHop(dspy.Module):\n",
+ " def __init__(self, hops, num_passages_to_retrieve, max_passages_in_context):\n",
+ " super().__init__()\n",
+ " self.hops = hops\n",
+ " self.num_passages_to_retrieve = num_passages_to_retrieve\n",
+ " self.max_passages_in_context = max_passages_in_context\n",
+ " self.retrieve = dspy.Retrieve(k = self.num_passages_to_retrieve)\n",
+ " self.generate_query = dspy.ChainOfThought(\"context ,question->search_query\")\n",
+ " self.generate_answer = dspy.ChainOfThought(\"context ,question->answer\")\n",
+ " self.generate_ranking = dspy.ChainOfThought(ReturnRankedDocuments)\n",
+ " \n",
+ " def forward(self,question):\n",
+ " context = []\n",
+ " full_context = []\n",
+ " top_context = []\n",
+ " max_passage_num = self.max_passages_in_context\n",
+ " for hop in range(self.hops):\n",
+ " # Get a new query\n",
+ " query = self.generate_query(context = context, question = question).search_query\n",
+ " # Get new passages\n",
+ " context = self.retrieve(query).passages\n",
+ " # Add these new passages to the previous top context \n",
+ " full_context = top_context + context\n",
+ " # Get the most important indices, ranked\n",
+ " most_important_indices = self.generate_ranking(question=question, context=full_context).ranking\n",
+ " indices = [int(num) for num in re.findall(r'\\d+', most_important_indices)]\n",
+ "\n",
+ " if len(indices) < max_passage_num:\n",
+ " indices = range(1,max_passage_num+1)\n",
+ "\n",
+ " valid_indices = [index-1 for index in indices if index-1 < len(context)]\n",
+ " top_indices = sorted(valid_indices, key=lambda x: x)[:max_passage_num+1]\n",
+ " most_important_context_list = [context[idx] for idx in top_indices]\n",
+ " # Save the top context\n",
+ " top_context = most_important_context_list\n",
+ "\n",
+ " return dspy.Prediction(context=context, answer=self.generate_answer(context = top_context , question = question).answer)\n",
+ "\n",
+ "program = RankingMultiHop(hops=4, num_passages_to_retrieve=5, max_passages_in_context=5)\n",
+ "\n",
+ "# Load and configure the datasets.\n",
+ "TRAIN_SIZE = 500\n",
+ "EVAL_SIZE = 500\n",
+ "\n",
+ "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)\n",
+ "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:TRAIN_SIZE]\n",
+ "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",
+ "\n",
+ "# Set up metrics\n",
+ "NUM_THREADS = 10\n",
+ "\n",
+ "metric = dspy.evaluate.answer_exact_match\n",
+ "\n",
+ "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=None)\n",
+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",
+ "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 2] Baseline Evaluation\n",
+ "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __16%__ on our trainset, and __21.4%__ on our devset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 80 / 500 (16.0): 100%|██████████| 500/500 [00:30<00:00, 16.32it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 80 / 500 (16.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 107 / 500 (21.4): 100%|██████████| 500/500 [00:29<00:00, 17.01it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 107 / 500 (21.4%)\n"
+ ]
+ }
+ ],
+ "source": [
+ "baseline_train_score = evaluate(program,devset=trainset)\n",
+ "baseline_eval_score = evaluate(program, devset=devset)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 3] Optimizing with MIPRO"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### 3a] Compile Program"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import cloudpickle as pickle\n",
+ "from dspy.teleprompt import BayesianSignatureOptimizer\n",
+ "\n",
+ "LOAD_PRECOMPILED_PROGRAM = True\n",
+ "\n",
+ "# By default, we will load the precompiled program\n",
+ "if LOAD_PRECOMPILED_PROGRAM:\n",
+ " # Load a our precompiled program\n",
+ " with open(compiled_program_file_path, 'rb') as file:\n",
+ " # Load the data from the file\n",
+ " compiled_program = pickle.load(file)\n",
+ "# Otherwise, if desired, the program can be compiled from scratch \n",
+ "else:\n",
+ " # Define hyperparameters:\n",
+ " N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",
+ " trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",
+ " temperature = 1.0 # The temperature configured for generating new instructions\n",
+ "\n",
+ " # Compile\n",
+ " eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",
+ " teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=True)\n",
+ " compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "generate_query = ChainOfThought(StringSignature(context, question -> search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ "))\n",
+ "generate_answer = ChainOfThought(StringSignature(context, question -> answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ "))\n",
+ "generate_ranking = ChainOfThought(ReturnRankedDocuments(question, context -> ranking\n",
+ " instructions='Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'desc': \"The question we're trying to answer.\", '__dspy_field_type': 'input', 'prefix': 'Question:'})\n",
+ " context = Field(annotation=str required=True json_schema_extra={'desc': 'List of potentially related passages.', '__dspy_field_type': 'input', 'prefix': 'Context:'})\n",
+ " ranking = Field(annotation=str required=True json_schema_extra={'desc': 'A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.', '__dspy_field_type': 'output', 'prefix': 'Ranking:'})\n",
+ "))"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "compiled_program"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### 3b] Evaluate optimized program"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 84 / 201 (41.8): 40%|████ | 200/500 [00:13<00:19, 15.48it/s]"
+ ]
+ }
+ ],
+ "source": [
+ "bayesian_train_score = evaluate(compiled_program, devset=trainset)\n",
+ "bayesian_eval_score = evaluate(compiled_program, devset=devset)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### 3c] Visualizing scores & prompts over trials"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, let's take a look at how this optimization looked over the course of each trial. We see that, in general, performance increases as trials go on, until it saturates after ~trial 13. Note that some of the 'pruned' trials have high scores, but were pruned early because they had comparitively lower scores on the easier slices of the data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAE8CAYAAACrYErbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABP3UlEQVR4nO3deVxUZfs/8M8MssOAKJvCACoCKmjgEiUIioILoWBk+JS4ZkFBpKX1mGiLS08FmsvzlKlZmEtom6KIoqBoSq45kRI0pqBGyYAoDDP37w9/c76OwwAzDMzAXO/Xi9fLOeeeM9c158jFuc99n8NjjDEQQgghRAVf3wEQQgghhoqKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJGlWWFgYwsLCtHovj8dDenq6TuMxZpp8nzweD8nJye0bUAfbsmULeDweysvL9R0KMSJUJLs4Ho/Xqp/8/Hy9xXj79m2kpKTA19cXlpaWcHJywvDhw/HGG2+gtrZWb3EZuhMnTiA9PR137tzR6XbLy8uVjg0TExMIhUJMmTIF586d0+lndSaFhYUYP348evfuDQsLCwiFQkRHRyMrK4trU1dXh/T09Db9f2qv/Uq0003fAZD2tW3bNqXXX3zxBXJzc1WW+/n5Nfn+gwcPtltsAPD3339j6NChkEgkmDVrFnx9fVFVVYULFy5gw4YNePHFF2FjY9OuMXQW9+7dQ7du//df9sSJE1i2bBkSExNhb2+v88979tlnMWHCBMhkMohEImzYsAH79+/HyZMnMWTIEJ1/Xkuee+45TJs2Debm5h3+2bt27cIzzzyDIUOGICUlBd27d0dZWRmOHTuGTz/9FAkJCQAeFMlly5YBgNY9MO29X4lmqEh2cf/617+UXp88eRK5ubkqyx9VV1cHKysrmJmZtWd42LRpE8RiMY4fP44nnnhCaZ1EImn3z3/Y3bt3YW1t3WGfpykLC4sO/bzAwECl4+TJJ5/EU089hQ0bNuC///1vk+9pz+/QxMQEJiYm7bLtlqSnp2PAgAE4efKkyjF569YtvcREOgZ1txKEhYVh0KBBKC4uRmhoKKysrPDmm29y6x7+i7ihoQFvv/02goKCYGdnB2tra4SEhODIkSNafXZpaSlMTEzw+OOPq6wTCAQqheHUqVOYMGECunfvDmtrawQEBCAzM1OpzeHDhxESEgJra2vY29sjJiYGIpFIqU16ejp4PB4uX76MhIQEdO/eHSNHjuTWf/nllwgKCoKlpSUcHBwwbdo0XLt2TWkbV65cQVxcHFxcXGBhYQE3NzdMmzYN1dXVavNds2YNTExMlLrSPvzwQ/B4PKSlpXHLZDIZbG1t8cYbb3DLHr4mmZ6ejoULFwIAvLy8uK7RR6/X7d27F4MGDYK5uTkGDhyInJwctbG1ZPTo0QCAsrIyAP93jfDo0aN46aWX4OTkBDc3NwBAYmIiPD09Vbah+N4fprh+2lKsTV2T9PT0xKRJk1BYWIjhw4fDwsICffr0wRdffKHy2RcuXMCoUaNgaWkJNzc3vPvuu9i8eXOrrnOWlpZi2LBhTf7R5uTkBOBBN7WjoyMAYNmyZdw+UeyzCxcuIDExEX369IGFhQVcXFwwa9YsVFVVKX0/6varoht8y5YtKjE8er26pqYGqamp8PT0hLm5OZycnDB27Fj8/PPPzeZJVNGZJAEAVFVVYfz48Zg2bRr+9a9/wdnZucl2EokEn332GZ599lnMnTsXNTU12LRpEyIjI/HTTz9p3A3n4eEBmUyGbdu2YcaMGc22zc3NxaRJk+Dq6oqUlBS4uLhAJBLhhx9+QEpKCgDg0KFDGD9+PPr06YP09HTcu3cPa9euxZNPPomff/5Z5Rf3008/DW9vb7z//vtQPDXuvffew5IlSxAfH485c+bg9u3bWLt2LUJDQ3H27FnY29ujoaEBkZGRqK+vx8svvwwXFxdcv34dP/zwA+7cuQM7O7smcwgJCYFcLkdhYSEmTZoEACgoKACfz0dBQQHX7uzZs6itrUVoaGiT24mNjcVvv/2G7du34+OPP0bPnj0BgPslDTy4hpadnY2XXnoJtra2WLNmDeLi4iAWi9GjR49mv+umlJaWAoDKe1966SU4Ojri7bffxt27dzXebltjvXr1KqZOnYrZs2djxowZ+Pzzz5GYmIigoCAMHDgQAHD9+nWEh4eDx+Nh8eLFsLa2xmeffdbqrlsPDw/k5eXhzz//5P4QeJSjoyN3iWDKlCmIjY0FAAQEBAB4cPz+/vvvmDlzJlxcXPDLL7/gf//7H3755RecPHkSPB6v2f16+/btVsUKAPPnz8fu3buRnJyMAQMGoKqqCoWFhRCJRAgMDGz1dggARoxKUlISe3S3jxo1igFgGzduVGk/atQoNmrUKO51Y2Mjq6+vV2rzzz//MGdnZzZr1iyl5QDY0qVLm42nsrKSOTo6MgDM19eXzZ8/n2VlZbE7d+4otWtsbGReXl7Mw8OD/fPPP0rr5HI59+8hQ4YwJycnVlVVxS07f/484/P57Pnnn+eWLV26lAFgzz77rNK2ysvLmYmJCXvvvfeUll+8eJF169aNW3727FkGgO3atavZ/B4lk8mYQCBgr7/+Ohd7jx492NNPP81MTExYTU0NY4yxjz76iPH5fKVcH/0+P/jgAwaAlZWVqXwOAGZmZsauXr2q9D0AYGvXrm02xrKyMgaALVu2jN2+fZtVVlay/Px89thjjzEA7JtvvmGMMbZ582YGgI0cOZI1NjYqbWPGjBnMw8NDZduK712bWBWf93C+Hh4eDAA7duwYt+zWrVvM3Nycvfbaa9yyl19+mfF4PHb27FluWVVVFXNwcFD7HT5s06ZNXJzh4eFsyZIlrKCggMlkMqV2t2/fVnvc19XVqSzbvn27Svzq9qtiv2zevFllO49+pp2dHUtKSmo2J9I61N1KAADm5uaYOXNmi+1MTEy4Lie5XI6///4bjY2NGDp0qFZdOc7Ozjh//jzmz5+Pf/75Bxs3bkRCQgKcnJzwzjvvcGd3Z8+eRVlZGVJTU1UGMyi67yoqKnDu3DkkJibCwcGBWx8QEICxY8di3759Kp8/f/58pdfZ2dmQy+WIj4/HX3/9xf24uLjA29ub61ZWnCkeOHAAdXV1rc6Xz+fjiSeewLFjxwAAIpEIVVVVWLRoERhjKCoqAvDg7HLQoEFtGrgRERGBvn37cq8DAgIgEAjw+++/t+r9S5cuhaOjI1xcXBAWFobS0lKsWrWKO0NSmDt3bpuvFbYl1gEDBiAkJIR77ejoCB8fH6X35uTkIDg4WKmnw8HBAdOnT29VfLNmzUJOTg7CwsJQWFiId955ByEhIfD29saJEydatQ1LS0vu3/fv38dff/3FXWbQdTeovb09Tp06hRs3buh0u8aIiiQBAPTu3bvVg2S2bt2KgIAAWFhYoEePHnB0dMSPP/7Y7LW45ri6umLDhg2oqKhASUkJ1qxZw3Xfbdq0CcD/dfUNGjRI7Xb++OMPAICPj4/KOj8/P/z1118q3YFeXl5Kr69cuQLGGLy9veHo6Kj0IxKJuEEaXl5eSEtLw2effYaePXsiMjIS69ata9V3EBISguLiYty7dw8FBQVwdXVFYGAgBg8ezHW5FhYWKv3i14ZQKFRZ1r17d/zzzz+tev+8efOQm5uLvLw8FBcX49atW3j99ddV2j36HXZ0rK157x9//IF+/fqptGtqmTqRkZE4cOAA7ty5g2PHjiEpKQl//PEHJk2a1KrBO3///TdSUlLg7OwMS0tLODo6ct+dtv931Fm9ejUuXboEd3d3DB8+HOnp6a3+44goo2uSBIDyX7nN+fLLL5GYmIjJkydj4cKFcHJygomJCVasWMEVMm3xeDz0798f/fv3x8SJE+Ht7Y2vvvoKc+bMadN2m/No3nK5HDweD/v372/y7Ojh6SgffvghEhMT8e233+LgwYN45ZVXsGLFCpw8eVLtdSsAGDlyJKRSKYqKilBQUMAVw5CQEBQUFODXX3/F7du321wk1Z3dKc7OW+Lt7Y2IiIgW2zV17Dw6OEdBJpM1ubwtsbY1T01ZWVkhJCQEISEh6NmzJ5YtW4b9+/e3eE09Pj4eJ06cwMKFCzFkyBDY2NhALpcjKioKcrm8xc/V5DuNj49HSEgI9uzZg4MHD+KDDz7AqlWrkJ2djfHjx7cuUQKAiiTR0O7du9GnTx9kZ2cr/addunSpTj+nT58+6N69OyoqKgCA64q7dOmS2l/cHh4eAICSkhKVdb/++it69uzZ4vSEvn37gjEGLy8v9O/fv8U4/f394e/vj3//+984ceIEnnzySWzcuBHvvvuu2vcMHz4cZmZmKCgoQEFBATeaMTQ0FJ9++iny8vK4181R90vTEHTv3r3JyfCKs/2O5uHhgatXr6osb2qZJoYOHQoA3HGqbp/8888/yMvLw7Jly/D2229zy69cuaLSVt02unfvDgAq36u679TV1RUvvfQSXnrpJdy6dQuBgYF47733qEhqiLpbiUYUf7U//Ff6qVOnuGtpmjp16lSTIyJ/+uknVFVVcV2ngYGB8PLyQkZGhsovCUUsrq6uGDJkCLZu3arU5tKlSzh48CAmTJjQYjyxsbEwMTHBsmXLVM5EGGPccH2JRILGxkal9f7+/uDz+aivr2/2MywsLDBs2DBs374dYrFY6Uzy3r17WLNmDfr27QtXV9dmt6Mo+IZ4Z5a+ffuiuroaFy5c4JZVVFRgz549eoknMjISRUVFSncM+vvvv/HVV1+16v2KP1wepbjOrThOraysAKjuk6b+3wBARkaGyjbV7VeBQICePXty17MV1q9fr/RaJpOpdN86OTmhV69eLR6bRBWdSRKNTJo0CdnZ2ZgyZQomTpyIsrIybNy4EQMGDNDqFnLbtm3DV199hSlTpiAoKAhmZmYQiUT4/PPPYWFhwc3X5PP52LBhA6KjozFkyBDMnDkTrq6u+PXXX/HLL7/gwIEDAIAPPvgA48ePR3BwMGbPns1NAbGzs2vVfU/79u2Ld999F4sXL0Z5eTkmT54MW1tblJWVYc+ePZg3bx4WLFiAw4cPIzk5GU8//TT69++PxsZGbNu2DSYmJoiLi2vxc0JCQrBy5UrY2dnB398fwINfZD4+PigpKUFiYmKL2wgKCgIAvPXWW5g2bRpMTU0RHR1tEDdEmDZtGt544w1MmTIFr7zyCurq6rBhwwb0799fL3P1Xn/9dXz55ZcYO3YsXn75ZW4KiFAoxN9//93iWXlMTAy8vLwQHR2Nvn374u7duzh06BC+//57DBs2DNHR0QAedD0PGDAAO3bsQP/+/eHg4IBBgwZh0KBBCA0NxerVqyGVStG7d28cPHiQm3P6sOb265w5c7By5UrMmTMHQ4cOxbFjx/Dbb78pvb+mpgZubm6YOnUqBg8eDBsbGxw6dAinT5/Ghx9+qKNv1IjoZ1At0Rd1U0AGDhzYZPtHp4DI5XL2/vvvMw8PD2Zubs4ee+wx9sMPPzQ55B+tmAJy4cIFtnDhQhYYGMgcHBxYt27dmKurK3v66afZzz//rNK+sLCQjR07ltna2jJra2sWEBCgMqXh0KFD7Mknn2SWlpZMIBCw6OhodvnyZaU2iqkIt2/fbjKub775ho0cOZJZW1sza2tr5uvry5KSklhJSQljjLHff/+dzZo1i/Xt25dZWFgwBwcHFh4ezg4dOtRsvgo//vgjA8DGjx+vtHzOnDkMANu0aZPKe5r6Pt955x3Wu3dvxufzlaYNAGhyCoCHhwebMWNGs7Epphp88MEHzbZTTMk4ffp0k+sPHjzIBg0axMzMzJiPjw/78ssv1U4BaU2s6qaATJw4UeW9jx63jD2YthMSEsLMzc2Zm5sbW7FiBVuzZg0DwCorK5vNdfv27WzatGmsb9++zNLSkllYWLABAwawt956i0kkEqW2J06cYEFBQczMzExpn/35559sypQpzN7entnZ2bGnn36a3bhxQ6P9WldXx2bPns3s7OyYra0ti4+PZ7du3VLaRn19PVu4cCEbPHgw9/9k8ODBbP369c3mSJrGY6ydrm4TQoiBS01NxX//+1/U1tbq7ZZ3xLDRNUlCiFG4d++e0uuqqips27YNI0eOpAJJ1KJrkoQQoxAcHIywsDD4+fnh5s2b2LRpEyQSCZYsWaLv0IgBoyJJCDEKEyZMwO7du/G///0PPB4PgYGB2LRpU4tTbYhxo2uShBBCiBp0TZIQQghRg4okIYQQokaXvyYpl8tx48YN2NraGvRtvAghhLQfxhhqamrQq1cv8PmtPz/s8kXyxo0bcHd313cYhBBCDMC1a9eafQDBo7p8kbS1tQXw4IsRCARab0cqleLgwYMYN24cTE1NdRWewTK2fAHK2RhyNrZ8AcpZkbNEIoG7uztXE1qryxdJRRerQCBoc5G0srKCQCAwigPN2PIFKGdjyNnY8gUo50dz1vSyGw3cIYQQQtSgIkkIIYSoQUWSEEK6KLlcDrFYDAAQi8WQy+V6jqjz6fLXJAkhxBiJRCLk5OSgtrYWAQEByMrKgo2NDaKiouDn56fv8DoNOpMkhJAuRiQSYefOnZBIJErLJRIJdu7cCZFIpKfIOh8qkoQQ0oXI5XLk5OQ02yYnJ4e6XluJiiQhhHQhYrFY5QzyURKJhLtWSZpHRZIQQrqQmpoanbYzdlQkCSGkC2ntHWU0vfOMsaIiSQghXYhQKGzx7mICgQBCobCDIurcqEgSQkgXwufzERUV1WybqKgojZ6EYczoWyKEkC7Gz88P8fHxKmeUAoEA8fHxNE9SA3QzAUII6YL8/Pzg4+ODsrIyXLp0CQkJCfDy8qIzSA3Rt0UIIV0Un8/nrj0KhUIqkFqgM0lCDJBMLkOBuAAVNRVwtXVFiDAEJnwTfYdFiNGhIkmIgckWZSMlJwV/Sv7klrkJ3JAZlYlYv1g9RkaI8aFzb0IMSLYoG1N3TlUqkABwXXIdU3dORbYoW0+REaIbcrkc5eXluHjxIsrLyw3+9nh0JkmIgZDJZUjJSQEDU1nHwMADD6k5qYjxiaGuV9IpKZ5M8vBt8wQCgUE/mYTOJAkxEAXiApUzyIcxMFyTXEOBuKADoyJENzrrk0noTJJ0SZoOfJHJZSgUFwIACsWFCPUKbbG9pttvqX1FTUWrcmttO10zxsFExphze2jtk0l8fHwMbgQuFUnS5Wg68EXRvqq2CtsDtmNi1kT0sOnRYntNt99Se1db11bl19p2umSMg4mMMef2osmTSTw9PTsmqFYyrJJNSBtpOvDFkNqHCEPgJnADD7wmc+OBB3eBO0KEIc18A7pnjIOJtM1ZJpchvzwf2y9uR355PmRyWUeEa/A685NJqEiSLqOlgS8AkJqTyv3iMrT2JnwTZEZlAoBKoVS8zojK6NDuPk1z6Aq0zTlblA3PTE+Ebw1HQnYCwreGwzPTs9k/IoylqHbmJ5NQkSRdhqYDXwytPQDE+sVid/xu9Bb0VmrrJnDD7vjdHd7NZ4yDibTJWZszT22KamfVmZ9MQkWSdBmaDnwxtPYKsX6xKE8px5EZR5AVm4UjM46gLKVML9fBDH0wUXvQNGdtzjyNrQu7Mz+ZxPAiIkRLmg58MbT2DzPhmyDMMwzP+j+LMM8wvY2oNOTBRO1F05w1PfM0xi5soPM+mYRGt5IuQzHw5brkepO/gHjgwU3gxg18MbT2hqgr5KApTXPW9MxTk6Ia5hmmeQIGTPFkErFYjJqaGtja2hr8jdcNJrKVK1eCx+MhNTWVW3b//n0kJSWhR48esLGxQVxcHG7evKm/IIlB03Tgi6G1N0RdIQdNaZqzpmeextiF/TA+nw9PT0/4+/vD09PToAskYCBF8vTp0/jvf/+LgIAApeWvvvoqvv/+e+zatQtHjx7FjRs3EBtL85OIepoOfDG09oaoK+SgKU1y1nTqjjF2YXdmeu9ura2txfTp0/Hpp5/i3Xff5ZZXV1dj06ZNyMrKwujRowEAmzdvhp+fH06ePInHH39cXyETAxfrF4sYn5hW3ylF0f5Y2TFILknwY8KPzd5xR9vtd+Y7t3SFHDTV2pwVZ55Td04FDzylLtqmzjyNsQu7M9N7kUxKSsLEiRMRERGhVCSLi4shlUoRERHBLfP19YVQKERRUZHaIllfX4/6+nruteIuD1KpFFKpVOs4Fe9tyzY6k66Q75O9n+T+LZfJIZc1/7SBEa4jkHspFyNcR7Sqvabb17R9R9B0P7dnDnK5HH/++Sdqa2thY2MDNzc3nXfFaXNctybn6H7R2B23G28cegPXa65zy91s3bAyYiWi+0UrfWbm2Ew8t+c5AGi6qI7N0Nn32xX+L2uqqZy1zZ/HGFP9U6aDfP3113jvvfdw+vRpWFhYICwsDEOGDEFGRgaysrIwc+ZMpYIHAMOHD0d4eDhWrVrV5DbT09OxbNkyleVZWVmwsrJqlzwIIYQYtrq6OiQkJKC6urrFOZsP09uZ5LVr15CSkoLc3FxYWFjobLuLFy9GWloa91oikcDd3R3jxo3T6It5lFQqRW5uLsaOHQtTU1NdhGrQjC1fgHLWV84lJSXYs2eP2vVTpkyBj4+PTj7LEPJ9mEwuQ9GfRaisrYSLjQuC3YJ13oVtaDl3hKZybuneserorUgWFxfj1q1bCAwM5JbJZDIcO3YMn3zyCQ4cOICGhgbcuXMH9vb2XJubN2/CxcVF7XbNzc1hbm6ustzU1FQnB4iuttNZGFu+AOXckeRyOXJzc5t98G5ubi4GDBig065XQ9nHpjBFeN/wjvksA8m5Iz2cs7a5621065gxY3Dx4kWcO3eO+xk6dCimT5/O/dvU1BR5eXnce0pKSiAWixEcHKyvsAkhOqTJ0yEI0Qe9nUna2tpi0KBBSsusra3Ro0cPbvns2bORlpYGBwcHCAQCvPzyywgODqaRrYR0EZ356RDEOOh9dGtzPv74Y/D5fMTFxaG+vh6RkZFYv369vsMihOhIZ346BDEOBlUk8/PzlV5bWFhg3bp1WLdunX4CIoS0K8XTIZrrcjXUp0MQ42AQd9whhBinzvx0CGIc6MgjhOhVZ306BDEOBtXdSggxTp3x6RD6IJfL6TvqYFQkSacgk8uM6r6hxkjxdAjSNJFIhJycHKXrtwKBAFFRUXS23Y6oSBKDly3KRkpOitIz+NwEbsiMyuyST6Ag5FEikQg7d+5UWS6RSLBz507qlm5HdJ5ODFq2KBtTd05VeUjtdcl1TN05FdmibD1FRkjHkMvlyMnJabZNTk5Os3ctItqjIkkMlkwuQ0pOSpOPE1IsS81JhUwu6+jQCOkwdFci/aIiSQxWgbhA5QzyYQwM1yTXUCAu6MCoCOlYdFci/aIiSQxWRU2FTtsR0hnRXYn0i4okMViutq46bUeUyeQyFIoLAQCF4kLqtjZQirsSNYfuStR+qEgSgxUiDIGbwI17WvujeODBXeCOEGFIB0fW+WWLsuGZ6YmJWRMBABOzJsIz05MGQhkguiuRftG3SgyWCd8EmVGZAKBSKBWvM6IyaL7k/yeXy1FeXo6LFy+ivLxc7WhHYx0xrJiIDzwYDNOZRoPSXYn0h+ZJEoMW6xeL3fG7m5wnmRGVQfMk/7/WTjRvacQwDzyk5qQixiemS/3xofh+amtrERAQgKysLNjY2HSqifh0VyL9oCJJDF6sXyxifGLojjtqaDLRXJMRw2GeYe0Vcod6+Pt5uKB0xon4dFeijkdFknQKJnyTLvNLW5daO9Hcx8cHfD7f6EYMa/r9EPIoOioI6cQ0nWhubCOGaSI+aSsqkoR0YppONDe2EcM0EZ+0FRVJQjoxTSeaG9uIYZqIT9qKiiQhnZg2E80VI4Z7C3ortXMTuGF3/O4uNWKYJuKTtqIiSUgnpu1E81i/WJSnlOPHhB8BAD8m/IiylLIuVSABmohP2o6ODEI6OW0nmpvwTTBSOBIAMFI4sst0sT6KJuKTtqApIIR0ATTRvHmK76esrAyXLl1CQkICvLy86PshLaIiSUgXQRPNm8fn8yEUCnHp0iX6A4K0Gh0lhBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGzZM0ADK5zOgeKGyMORNCOh8qknqWLcpGSk6K0tPi3QRuyIzKVHsfzc5eYLTJmRBC9IG6W/UoW5SNqTunKhULALguuY6pO6ciW5Td5Hs8Mz0RvjUcCdkJCN8aDs9MzybbGiJtciaEEH2hIqknMrkMKTkpYGAq6xTLUnNSIZPLuOWdvcBokzMhhOgTFUk9KRAXqBS7hzEwXJNcQ4G4AEDXKDCa5kwIIfqm1yK5YcMGBAQEQCAQQCAQIDg4GPv37+fW379/H0lJSejRowdsbGwQFxeHmzdv6jHi1pHJZcgvz8f2i9uRX57fZOGqqKlo1bYU7bpCgdE0Z0II0Te9Dtxxc3PDypUr4e3tDcYYtm7dipiYGJw9exYDBw7Eq6++ih9//BG7du2CnZ0dkpOTERsbi+PHj+sz7Ga1dlCKq61rq7anaNcVCoymORNCiL7p9UwyOjoaEyZMgLe3N/r374/33nsPNjY2OHnyJKqrq7Fp0yZ89NFHGD16NIKCgrB582acOHECJ0+e1GfYamlyzTBEGAI3gRt44DW5LR54cBe4I0QYAqBrFBhNcyaEEH0zmCkgMpkMu3btwt27dxEcHIzi4mJIpVJERERwbXx9fSEUClFUVITHH3+8ye3U19ejvr6eey2RSAAAUqkUUqlU6/gU71W3DZlchjcOvAELvkWT63ngYdGBRZjQZwI3XSNzbCae2/McAChda1QUkYyxGZDL5JDL5Hjc9XH0s+uHGzU3mrwuyQMPvW1743HXx9uUp0JL+WpLk5w7WnvlbMiMLWdjyxegnB9dpikeY0z1N24HunjxIoKDg3H//n3Y2NggKysLEyZMQFZWFmbOnKlU8ABg+PDhCA8Px6pVq5rcXnp6OpYtW6ayPCsrC1ZWVu2SAyGEEMNWV1eHhIQEVFdXQyAQtPp9ej+T9PHxwblz51BdXY3du3djxowZOHr0qNbbW7x4MdLS0rjXEokE7u7uGDdunEZfzKOkUilyc3MxduxYmJqaqqzffXk3Zn83u8XtbHpqE6YOmKq0TCaXoejPIlTWVsLFxgXBbsFqbw7wfcn3eOPQG7hec51b5mbrhpURKxHtE61hVqpKSkpw6NAh3L17F4MGDcKlS5dgbW2NiIgI+Pj4tHn7Cprk3FFa2sddUWfNWXGc1tTUcMtsbW1bPE47a75tQTk/yFnRq6gpvRdJMzMz9OvXDwAQFBSE06dPIzMzE8888wwaGhpw584d2Nvbc+1v3rwJFxcXtdszNzeHubm5ynJTU1OdHCDqtuNq54p78nstvt/VzlXl/aYwRXjf8FZ9fuygWMQMiGmXO+6IRCJ88803AAA+/8HlarlcjurqanzzzTeIj4+Hn59fmz8H0CznjqarY6Uz6Uw5P3ycPkyT47Qz5asrxp6ztrkb3DxJuVyO+vp6BAUFwdTUFHl5edy6kpISiMViBAcH6zHCpnXkoBQTvgnCPMPwrP+zCPMM00mBlMvlyMnJabZNTk4O5PKOv1ZIiAIdp6Sj6bVILl68GMeOHUN5eTkuXryIxYsXIz8/H9OnT4ednR1mz56NtLQ0HDlyBMXFxZg5cyaCg4PVDtrRJxO+CTKjMgFApVByg1KiMvTepaiOWCxusTtCIpFALBZ3UESEqKLjlHQ0vXa33rp1C88//zwqKipgZ2eHgIAAHDhwAGPHjgUAfPzxx+Dz+YiLi0N9fT0iIyOxfv16fYbcrFi/WOyO393kPMmMqAyDvnn3w9d2dNGOELlcDrFYjJqaGtja2kIoFHLd+Nqi49T4tMdxpAm9FslNmzY1u97CwgLr1q3DunXrOiiitov1i0WMT/tcM2xPtra2Om1HjJtIJEJOTo7SWZ9AIEBUVFSbrmvTcWpc2us40oTBXZPsCtrjmmF7EwqFLY7+FQgEEAqFHRQR6axEIhF27typ0i0qkUiwc+dOiEQirbdNx6nxaM/jSBNUJAmAB6NZo6Kimm0TFRXVod0cpPNp74E1dJwaB0MaoEVHEuH4+fkhPj5e5S91gUCg0+kfpOvqiIE1dJx2fYY0QEvv8ySJYfHz84OPjw/Kyspw6dIlJCQkwMvLi/4yJ63SUQNrFMepPgd0kPZjSAO0qEgSFXw+H0KhEJcuXaJfPEQjHTmwhs/nw9PTs83bIYbHkAZo0W8/QojO0MAaoguGdBxRkSSE6AwNrCG6YEjHER2phBCdooE1RBcM5Tiia5KEEJ2jgTVEFwzhOKIiSQhpFzSwhuiCvo8j+rOOEEIIUaNNRbKhoQElJSVobGzUVTyEEEKIwdCqSNbV1WH27NmwsrLCwIEDubsevPzyy1i5cqVOAySEEEL0RasiuXjxYpw/fx75+fmwsLDglkdERGDHjh06C44QQgjRJ60G7uzduxc7duzA448/Dh7v/x4wPHDgQJSWluosOEIIIUSftDqTvH37NpycnFSW3717V6loEkIIIZ2ZVkVy6NCh+PHHH7nXisL42WefITg4WDeRGRCZXIZCcSEAoFBcCJlcpueICCGEdAStulvff/99jB8/HpcvX0ZjYyMyMzNx+fJlnDhxAkePHtV1jHqVLcpGSk4KqmqrsD1gOyZmTUQPmx7IjMpErF+svsMjhBDSjrQ6kxw5ciTOnz+PxsZG+Pv74+DBg3ByckJRURGCgoJ0HaPeZIuyMXXnVPwp+VNp+XXJdUzdORXZomw9RUYIIaQjaHwmKZVK8cILL2DJkiX49NNP2yMmgyCTy5CSkwIGprKOgYEHHlJzUhHjEwMTvokeIiSEENLeND6TNDU1xTfffNMesRiUAnGByhnkwxgYrkmuoUBc0IFREUII6UhadbdOnjwZe/fu1XEohqWipkKn7QghhHQ+Wg3c8fb2xvLly3H8+HEEBQXB2tpaaf0rr7yik+D0ydXWVaft9Ekml6FAXICKmgq42roiRBhCXcSEENIKWhXJTZs2wd7eHsXFxSguLlZax+PxukSRDBGGwE3ghuuS601el+SBBzeBG0KEIXqIrvUUo3Mf7jp2E7jR6FxCCGkFrYpkWVmZruMwOCZ8E2RGZWLqzqngQfkGCYrXGVEZBn1Gphid+2iRV4zO3R2/mwolIYQ0o82PymKMgTHVM62uINYvFrvjd6O3oLfScjeBm8EXmJZG5wJAak4q3RiBEEKaoXWR/OKLL+Dv7w9LS0tYWloiICAA27Zt02VsBiHWLxblKeX4MeHBHYZ+TPgRZSllBl0gARqdSwjRnFwuR3l5OS5evIjy8nLI5XKdtu+MtOpu/eijj7BkyRIkJyfjySefBAAUFhZi/vz5+Ouvv/Dqq6/qNEh9M+GbYKRwJPZd2oeRwpEG3cWqQKNzCSGaEIlEyMnJgUQi4ZYJBAJERUXBz8+vze07K62K5Nq1a7FhwwY8//zz3LKnnnoKAwcORHp6epcrkp1RVxqdSwhpXyKRCDt37lRZLpFIsHPnTsTHxysVPk3bd2ZadbdWVFTgiSeeUFn+xBNPoKKCzkwMgWJ07qODjhR44MFd4G7wo3MJIe1LLpcjJyen2TY5OTlcV6qm7Ts7rYpkv379mvwrYseOHfD29m5zUKTtFKNzAXTa0bmEkPYnFouVukybIpFIIBaLtWrf2WnV3bps2TI888wzOHbsGHdN8vjx48jLy2uyeBL9UIzObWqeZEZUhsEPPiKEtL+amhqN2mnavrPTqkjGxcXh1KlT+Pjjj7nb0/n5+eGnn37CY489psv4SBvF+sUixieG7rhDCGmSra2tRu00bd/ZaVUkASAoKAhffvmlLmMh7cSEb4IwzzB9h0EIMUBCoRACgaDZLlSBQAChUKhV+85Oq2uS+/btw4EDB1SWHzhwAPv3729zUIQQQjoGn89HVFRUs22ioqLA5/O1at/ZaZXFokWLIJOp3qmFMYZFixa1OShCCCEdx8/PD/Hx8RAIBErLBQJBk9M5NG3fmWnV3XrlyhUMGDBAZbmvry+uXr3a6u2sWLEC2dnZ+PXXX2FpaYknnngCq1atgo+PD9fm/v37eO211/D111+jvr4ekZGRWL9+PZydnbUJnRBCSBP8/Pzg4+MDsViMmpoa2NraQigUqj0j1LR9Z6VVNnZ2dvj9999Vll+9elXlsVnNOXr0KJKSknDy5Enk5uZCKpVi3LhxuHv3Ltfm1Vdfxffff49du3bh6NGjuHHjBmJjaVQmIYToGp/Ph6enJ/z9/eHp6dliwdO0fWek1ZlkTEwMUlNTsWfPHvTt2xfAgwL52muv4amnnmr1dh6dkLplyxY4OTmhuLgYoaGhqK6uxqZNm5CVlYXRo0cDADZv3gw/Pz+cPHkSjz/+uDbhE0IIIa2iVZFcvXo1oqKi4OvrCzc3NwDAtWvXEBoaiv/85z9aB1NdXQ0AcHBwAAAUFxdDKpUiIiKCa+Pr6wuhUIiioqImi2R9fT3q6+u514oRWFKpFFKpVOvYFO9tyzY6E2PLF6CcjYGx5QtQzo8u0xSPafmcK8YYcnNzcf78eVhaWmLw4MEICdH+FmdyuRxPPfUU7ty5g8LCQgBAVlYWZs6cqVT0AGD48OEIDw/HqlWrVLaTnp6OZcuWqSzPysqClZWV1vERQgjpvOrq6pCQkIDq6mqVAUfN0ehMsqioCFVVVZg0aRJ4PB7GjRuHiooKLF26FHV1dZg8eTLWrl0Lc3NzjRNISkrCpUuXuAKprcWLFyMtLY17LZFI4O7ujnHjxmn0xTxKKpUiNzcXY8eOhampaZti7AyMLV+AcjaGnI0tX4ByVuTc0q301NGoSC5fvhxhYWGYNGkSAODixYuYO3cuZsyYAT8/P3zwwQfo1asX0tPTNQoiOTkZP/zwA44dO8Z13wKAi4sLGhoacOfOHdjb23PLb968CRcXlya3ZW5u3mSRNjU11ckBoqvtdBbGli9AORsDY8sXoJy1zV2joUjnzp3DmDFjuNdff/01hg8fjk8//RRpaWlYs2aNRvduZYwhOTkZe/bsweHDh+Hl5aW0PigoCKampsjLy+OWlZSUQCwWIzg4WJPQCSGEEI1pdCb5zz//KM1PPHr0KMaPH8+9HjZsGK5du9bq7SUlJSErKwvffvstbG1tUVlZCeDBFBNLS0vY2dlh9uzZSEtLg4ODAwQCAV5++WUEBwfTyFZCCCHtTqMzSWdnZ5SVlQEAGhoa8PPPPysVq5qaGo1OaTds2IDq6mqEhYXB1dWV+9mxYwfX5uOPP8akSZMQFxeH0NBQuLi4IDs7W5OwCSGEEK1odCY5YcIELFq0CKtWrcLevXthZWWlNKL1woUL3LzJ1mjNwFoLCwusW7cO69at0yRUQgghpM00KpLvvPMOYmNjMWrUKNjY2GDr1q0wMzPj1n/++ecYN26czoMkhBBC9EGjItmzZ08cO3YM1dXVsLGxgYmJ8jMJd+3aBRsbG50GSAghhOiLVnfcsbOza3K54k45hBBCSFfQ9e5GSwghhOgIFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKKGXovksWPHEB0djV69eoHH42Hv3r1K6xljePvtt+Hq6gpLS0tERETgypUr+gmWEEKI0dFrkbx79y4GDx6MdevWNbl+9erVWLNmDTZu3IhTp07B2toakZGRuH//fgdHSgghxBh10+eHjx8/HuPHj29yHWMMGRkZ+Pe//42YmBgAwBdffAFnZ2fs3bsX06ZN68hQCSGEGCG9FsnmlJWVobKyEhEREdwyOzs7jBgxAkVFRWqLZH19Perr67nXEokEACCVSiGVSrWOR/HetmyjMzG2fAHK2RgYW74A5fzoMk0ZbJGsrKwEADg7Oystd3Z25tY1ZcWKFVi2bJnK8oMHD8LKyqrNceXm5rZ5G52JseULUM7GwNjyBSjnuro6rbZhsEVSW4sXL0ZaWhr3WiKRwN3dHePGjYNAINB6u1KpFLm5uRg7dixMTU11EapBM7Z8AcrZGHI2tnwBylmRs6JXUVMGWyRdXFwAADdv3oSrqyu3/ObNmxgyZIja95mbm8Pc3FxluampqU4OEF1tp7MwtnwBytkYGFu+AOWsbe4GWyS9vLzg4uKCvLw8rihKJBKcOnUKL774on6DI4RoTCaT6f26mFQqRbdu3XD//n3IZDK9xtJRjClnMzMz8Pm6nbSh1yJZW1uLq1evcq/Lyspw7tw5ODg4QCgUIjU1Fe+++y68vb3h5eWFJUuWoFevXpg8ebL+giaEaIQxhsrKSty5c0ffoYAxBhcXF1y7dg08Hk/f4XQIY8qZz+fDy8tLp3nqtUieOXMG4eHh3GvFtcQZM2Zgy5YteP3113H37l3MmzcPd+7cwciRI5GTkwMLCwt9hUwI0ZCiQDo5OcHKykqvv6jlcjlqa2thY2Oj8zMOQ2UsOcvlcty4cQMVFRVKl+jaSq9FMiwsDIwxtet5PB6WL1+O5cuXd2BUhBBdkclkXIHs0aOHvsOBXC5HQ0MDLCwsunTBeJgx5ezo6IgbN27otFu5a39jhBC9UlyD1MX0K0JaYmZmBgBUJAkhnUtXvxZGDIPiOGuuh1JTVCQJIYQQNahIEkKIETp+/Dj8/f1hamra6hkD6enpzc5T10ZTT4Dq6BiaQ0WSEEKacPv2bbz44osQCoUwNzeHi4sLIiMjcfz4cX2HphNpaWkYMmQIysrKsGXLljZvLz09HTwer9mfplRUVKh90IUhMNibCRBCyMNkchkKxAWoqKmAq60rQoQhMOGbtNvnxcXFoaGhAVu3bkWfPn1w8+ZN5OXloaqqqt0+syOVlpZi/vz5cHNz08n2FixYgPnz53Ovhw0bhnnz5mHu3LlNtm9oaICZmRl3dzVDRWeShBCDly3KhmemJ8K3hiMhOwHhW8PhmemJbFF2u3zenTt3UFBQgFWrViE8PBweHh4YPnw4Fi9ejKeeegoAUF5eDh6Ph3Pnzim9j8fjIT8/n1v2yy+/YNKkSRAIBLC1tUVISAhKS0u59Z9//jkGDhwIc3NzuLq6Ijk5WWl7c+bMgaOjIwQCAUaPHo3z589z68+fP4/w8HDY2tpCIBAgKCgIZ86cAQD88ccfiI6ORo8ePdC7d2/4+/tj3759XNxVVVWYNWsWeDwetmzZgi1btsDe3l7pe9i7d2+rB13Z2NjAxcWF+zExMYGtrS33etq0aUhOTkZqaip69uyJyMhIAKrdrW+88Qb69+8PKysr9OnTB0uWLNHrnZqoSBJCDFq2KBtTd07Fn5I/lZZfl1zH1J1T26VQ2tjYwMbGBnv37lV69J6mrl+/jtDQUJibm+Pw4cMoLi7GrFmz0NjYCADYsGEDkpKSMG/ePFy8eBHfffcd+vXrx73/6aefxq1bt7B//34UFxcjMDAQY8aMwd9//w0AmD59Otzc3HD69GkUFxdj0aJF3D1Kk5KSUF9fj/z8fBw/fhwrVqyAjY0N3N3dUVFRAYFAgIyMDFRUVOCZZ55pw7fVelu3boWZmRmOHz+OjRs3NtnG1tYWW7ZsweXLl5GZmYlPP/0UH3/8cYfE1xTqbiWEGCyZXIaUnBQwqA7pZ2DggYfUnFTE+MTotOu1W7du2LJlC+bOnYuNGzciMDAQo0aNwrRp0xAQENDq7axbtw52dnb4+uuvueLVv39/bv27776L1157DSkpKdyyYcOGAQAKCwvx008/4datW9xDG/7zn/9g79692L17N+bNmwexWIyFCxfC19cXAODt7c1tRywWIy4uDv7+/pBIJAgICOBuJuDi4gIejwc7O7sO7e709vbG6tWrm23z73//m/u3p6cnFixYgK+//hqvv/56e4fXJDqTJIQYrAJxgcoZ5MMYGK5JrqFAXKDzz46Li8ONGzfw3XffISoqCvn5+QgMDNRokMu5c+cQEhLS5BMobt26hRs3bmDMmDFNvvf8+fOora1Fjx49uDNbGxsblJWVcd21aWlpmDNnDiIiIrBy5UqlbtxXXnkF7777LkJCQrBixQpcuHBBsy+gHQQFBbXYZseOHXjyySfh4uICGxsb/Pvf/4ZYLO6A6JpGRZIQYrAqaip02k5TFhYWGDt2LJYsWYITJ04gMTERS5cuBQDurOzhieuPXjuztLRUu+3m1gEPHgDh6uqKc+fOKf2UlJRg4cKFAB6MKP3ll18wceJEHD58GAMGDMCePXsAAHPmzMHvv/+O6dOn4/Llyxg+fDjWrl2r9vP4fL7KJHxdXwu0trZudn1RURGmT5+OCRMm4IcffsDZs2fx1ltvoaGhQadxaIKKJCHEYLnatu5G1a1t11YDBgzA3bt3ATy4TyjwYAqDwsODeAAgICAABQUFTRYbW1tbeHp6Ii8vr8nPCgwMRGVlJbp164Z+/fop/fTs2ZNr179/f7z66qs4ePAgYmNjsXnzZm6du7s75s+fj23btiEtLQ2ffvqp2twcHR1RU1PD5ddUPu3txIkT8PDwwFtvvYWhQ4fC29sbf/zxR4fG8CgqkoQQgxUiDIGbwA08ND3Ckgce3AXuCBGG6PRzq6qqMHr0aHz55Ze4cOECysrKsGvXLqxevRoxMTEAHpwJPv7441i5ciVEIhGOHj2qdD0NAJKTkyGRSDBt2jScOXMGV65cwbZt21BSUgLgwZnghx9+iDVr1uDKlSv4+eefubO9iIgIBAcHY/LkyTh48CDKy8tx4sQJvPXWWzhz5gzu3buH5ORk5Ofn448//sDx48dx+vRp+Pn5AQBSU1Nx4MABlJWV4fz588jPz+fWNWXEiBGwsrLCm2++idLSUmRlZelk/qQmvL29IRaL8fXXX6O0tBRr1qzhzoz1hYokIcRgmfBNkBmVCQAqhVLxOiMqQ+fzJW1sbDBixAh8/PHHCA0NxaBBg7BkyRLMnTsXn3zyCdfu888/R2NjI4KCgrjn3z6sR48eOHz4MGprazFq1CgEBQXh008/5a5RzpgxAxkZGVi/fj0GDhyISZMm4cqVKw/y4/Gwb98+hIaGYubMmejfvz+mTZuGP/74A87OzjAxMUFVVRWef/559O/fH/Hx8Rg/fjyWLVsG4MFNvpOSkjBw4EBMnToV3t7eWL9+vdqcHRwc8OWXX2Lfvn3w9/fH9u3bkZ6ertPvtSVPPfUUXn31VSQnJ2PIkCE4ceIElixZ0qExPIrHdHknWAMkkUhgZ2eH6upqCAQCrbcjlUqxb98+TJgwocmL8F2NseULUM7tkfP9+/dRVlYGLy+vNj0HNluUjZScFKVBPO4Cd2REZSDWL7bV25HL5ZBIJBAIBF3+sVEKxpSz4nhzc3PD4cOHlY5rbWsBTQEhhBi8WL9YxPjEdOgddwgBqEgSQjoJE74JwjzD9B0GMTJd+9ybEEIIaQMqkoQQQogaVCQJIYQQNahIEkIIIWpQkSSEEELUoCJJCCGEqEFFkhBCCFGDiiQhhHQRnp6eyMjI0Nn2tmzZAnt7e73GoG9UJAkhpAmJiYng8Xjg8XgwMzNDv379sHz5cjQ2Nuo7NK306dOHy6epn8TERJX3PPPMM/jtt986PlgDQnfcIYR0CnK5HGKxGDU1NbC1tYVQKGz3e5FGRUVh8+bNqK+vx759+5CUlARTU1MsXrxYpW1DQwPMzMzaNZ62OHXqFPe8yBMnTiAuLg4lJSXcfUwffb6lVCqFpaVli8+97OroTJIQYvBEIhEyMzOxdetWZGdnY+vWrcjMzIRIJGrXzzU3N4eLiws8PDzw4osvIiIiAt999x2AB2eakydPxnvvvYdevXrBx8cHwIOnd+zdu1dpO/b29txjp8rLy8Hj8ZCdnY3w8HBYWVlh8ODBKCoqUnpPYWEhQkJCYGlpCXd3d7zyyitKz3q8desWoqOjYWlpCS8vL3z11VfN5uLo6AgXFxe4uLjAwcEBAODk5AQXFxfcv38f9vb22LFjB0aNGgULCwt89dVXKt2tpaWliImJgbOzM2xsbDBs2DAcOnRIm6+206AiSQgxaCKRCDt37oREIlFaLpFIsHPnznYvlA+ztLREQ0MD9zovLw8lJSXIzc3FDz/8oNG23nrrLSxYsADnzp1D//798eyzz3JduaWlpYiKikJcXBwuXLiAHTt2oLCwEMnJydz7ExMTce3aNRw5cgS7d+/G+vXrcevWrTblt2jRIqSkpEAkEiEyMlJlfW1tLSZMmIC8vDycPXsWUVFRiI6OhlgsbtPnGjLqbiWEGCy5XI6cnJxm2+Tk5MDHx6ddu14ZY8jLy8OBAwfw8ssvc8utra3x2WefadXNumDBAkycOBEAsGzZMgwcOBBXr16Fr68vVqxYgenTpyM1NRXAg4cRr1mzBqNGjcKGDRsgFouxf/9+/PTTTxg2bBgAYNOmTc0+VLk1UlNTERur/tFjgwcPxuDBg7nX77zzDvbs2YPvvvtOqYB3JVQkCSEGSywWq5xBPkoikUAsFsPT01Pnn//DDz/AxsYGUqkUcrkcCQkJSg8i9vf31/o6ZEBAAPdvV1dXAA+6UH19fXH+/HlcuHBBqQuVMQa5XI6ysjL89ttv6NatG4KCgrj1vr6+Go9EfdTQoUObXV9bW4v09HT8+OOPqKioQGNjI+7du0dnkoQQog81NTU6baep8PBwbNiwAWZmZujVqxe6dVP+lWltba3yHh6Ph0efZS+VSlXaPfyQax6PB+DBmTPwoBi98MILeOWVV1TeJxQK223EaVP5PGzBggXIzc3Ff/7zH/Tr1w+WlpaYOnWqUhd0V0NFkhBisGxtbXXaTlPW1tbo16+fRu9xdHRERUUF9/rKlSuoq6vTaBuBgYG4fPmy2s/29fVFY2MjiouLue7WkpIS3LlzR6PP0dTx48eRmJiIKVOmAHhQzMvLy9v1M/WNBu4QQgyWUCjkpiioIxAIIBQKOyiilo0ePRqffPIJzp49izNnzmD+/PlKZ42t8cYbb+DEiRNITk7GuXPncOXKFXz77bfcdT8fHx9ERUXhhRdewKlTp1BcXIw5c+a0+3QNb29vZGdn49y5czh//jwSEhK4s9+uiookaTO5XI7y8nJcvHgR5eXlXf4/Dek4fD4fUVFRzbaJiopq9/mSmvjwww/h7u6OkJAQJCQkYMGCBbCystJoGwEBATh69Ch+++03hISE4LHHHsPbb7+NXr16cW02b96MXr16YdSoUYiNjcW8efPg5OSk63SUfPTRR+jevTueeOIJREdHIzIyEoGBge36mfrGY492nhugdevW4YMPPkBlZSUGDx6MtWvXYvjw4a16r0QigZ2dHaqrq1v8i7Q5UqkU+/btw4QJEzT+q7Azam2+IpEIOTk5SoMrBAIBoqKi2jzSrqMZ2z4G2j/n+/fvo6ysDF5eXrCwsNB6O7o6zuRyOSQSCQQCgUEV1vZkTDkrjjc3NzccPnxY6bjWthYY/DXJHTt2IC0tDRs3bsSIESOQkZGByMhIlJSUtPtfTaR5ivlrj1LMX4uPj+90hZIYJj8/P/j4+HT4HXcIMfgj7KOPPsLcuXMxc+ZMDBgwABs3boSVlRU+//xzfYdm1Fo7f426Xomu8Pl8eHp6wt/fH56enlQgSYcw6DPJhoYGFBcXK90nkc/nIyIiQuUWTgr19fWor6/nXiu6Z6RSaZPDsFtL8d62bKMzaSlfsViM2traZn9R1dbWoqyszKAGVTTH2PYx0P45S6VSbn6fIfzBpLi6pIjJGBhTznK5HIwx7s5FDx/X2h7jBn1N8saNG+jduzdOnDiB4OBgbvnrr7+Oo0eP4tSpUyrvSU9Px7Jly1SWZ2VlaXzxnBDSNt26dYOLiwvc3d0N+ubfpGtoaGjAtWvXUFlZqfK0lrq6OiQkJHS9a5KaWrx4MdLS0rjXEokE7u7uGDduXJsH7uTm5mLs2LFGMaijpXzFYjGysrJa3E5CQkKnOpM0pn0MtH/O9fX1EIvFsLa2NoinSTDGuGuaign8XZ0x5Xzv3j1YWlriiSeewLFjx5SO65bu3KSOQRfJnj17wsTEBDdv3lRafvPmTbi4uDT5HnNzc5ibm6ssNzU11ckvAV1tp7NQl6+XlxdsbGyaPfAEAgG8vLw63bUjY9vHQPvlzOfzwePxcP/+/Rbv5tIRFN2NPB6v0x2X2jKmnBsbG8Hj8biR1A8f19oe3wZdJM3MzBAUFIS8vDxMnjwZwIMdnpeX12VvpttZKOavNTW6VcHQ5q+RjmdiYgJ7e3vu6RRWVlZ6PZuRy+VoaGjA/fv3jebYNJac5XI5bt++DSsrK5iYmOhsuwZdJAEgLS0NM2bMwNChQzF8+HBkZGTg7t27mDlzpr5DM3p+fn6Ij4/vMvMkSftQ9Pq09TFOusAY47rkunrXo4Ix5czn8yEUCnWap8EXyWeeeQa3b9/G22+/jcrKSgwZMgQ5OTlwdnbWd2gENH+NtIzH48HV1RVOTk56HzkslUpx7NgxhIaGGk2XujHlbGZmBj6fr9PjzOCLJAAkJydT96oBU8xfI6Q5JiYmOu0G0zaGxsZGWFhYdPmCoWCMOesS/blPCCGEqEFFkhBCCFGDiiQhhBCiRqe4JtkWihsKaTuRVEEqlaKurg4SicQo+vWNLV+AcjaGnI0tX4ByfvRmApreZK7LF8mamhoAgLu7u54jIYQQom81NTWws7NrdXuDvnerLsjlcty4caPNt2RS3N7u2rVrbbq9XWdhbPkClLMx5Gxs+QKUsyJnxe35evXqpdEUtS5/Jsnn8+Hm5qaz7QkEAqM50ADjyxegnI2BseULUM4ANDqDVKCBO4QQQogaVCQJIYQQNahItpK5uTmWLl3a5BNGuiJjyxegnI2BseULUM5t1eUH7hBCCCHaojNJQgghRA0qkoQQQogaVCQJIYQQNahIEkIIIWpQkWyFdevWwdPTExYWFhgxYgR++uknfYfUbtLT08Hj8ZR+fH199R2WTh07dgzR0dHo1asXeDwe9u7dq7SeMYa3334brq6usLS0REREBK5cuaKfYHWgpXwTExNV9nlUVJR+gtWRFStWYNiwYbC1tYWTkxMmT56MkpISpTb3799HUlISevToARsbG8TFxeHmzZt6irhtWpNvWFiYyn6eP3++niJuuw0bNiAgIIC7YUBwcDD279/PrdfV/qUi2YIdO3YgLS0NS5cuxc8//4zBgwcjMjISt27d0ndo7WbgwIGoqKjgfgoLC/Udkk7dvXsXgwcPxrp165pcv3r1aqxZswYbN27EqVOnYG1tjcjISNy/f7+DI9WNlvIFgKioKKV9vn379g6MUPeOHj2KpKQknDx5Erm5uZBKpRg3bhzu3r3LtXn11Vfx/fffY9euXTh69Chu3LiB2NhYPUatvdbkCwBz585V2s+rV6/WU8Rt5+bmhpUrV6K4uBhnzpzB6NGjERMTg19++QWADvcvI80aPnw4S0pK4l7LZDLWq1cvtmLFCj1G1X6WLl3KBg8erO8wOgwAtmfPHu61XC5nLi4u7IMPPuCW3blzh5mbm7Pt27frIULdejRfxhibMWMGi4mJ0Us8HeXWrVsMADt69Chj7ME+NTU1Zbt27eLaiEQiBoAVFRXpK0ydeTRfxhgbNWoUS0lJ0V9QHaB79+7ss88+0+n+pTPJZjQ0NKC4uBgRERHcMj6fj4iICBQVFekxsvZ15coV9OrVC3369MH06dMhFov1HVKHKSsrQ2VlpdI+t7Ozw4gRI7r0Ps/Pz4eTkxN8fHzw4osvoqqqSt8h6VR1dTUAwMHBAQBQXFwMqVSqtJ99fX0hFAq7xH5+NF+Fr776Cj179sSgQYOwePFi1NXV6SM8nZPJZPj6669x9+5dBAcH63T/dvkbnLfFX3/9BZlMBmdnZ6Xlzs7O+PXXX/UUVfsaMWIEtmzZAh8fH1RUVGDZsmUICQnBpUuXYGtrq+/w2l1lZSUANLnPFeu6mqioKMTGxsLLywulpaV48803MX78eBQVFcHExETf4bWZXC5HamoqnnzySQwaNAjAg/1sZmYGe3t7pbZdYT83lS8AJCQkwMPDA7169cKFCxfwxhtvoKSkBNnZ2XqMtm0uXryI4OBg3L9/HzY2NtizZw8GDBiAc+fO6Wz/UpEkSsaPH8/9OyAgACNGjICHhwd27tyJ2bNn6zEy0l6mTZvG/dvf3x8BAQHo27cv8vPzMWbMGD1GphtJSUm4dOlSl7u2ro66fOfNm8f929/fH66urhgzZgxKS0vRt2/fjg5TJ3x8fHDu3DlUV1dj9+7dmDFjBo4eParTz6Du1mb07NkTJiYmKiOibt68CRcXFz1F1bHs7e3Rv39/XL16Vd+hdAjFfjXmfd6nTx/07NmzS+zz5ORk/PDDDzhy5IjSI/NcXFzQ0NCAO3fuKLXv7PtZXb5NGTFiBAB06v1sZmaGfv36ISgoCCtWrMDgwYORmZmp0/1LRbIZZmZmCAoKQl5eHrdMLpcjLy8PwcHBeoys49TW1qK0tBSurq76DqVDeHl5wcXFRWmfSyQSnDp1ymj2+Z9//omqqqpOvc8ZY0hOTsaePXtw+PBheHl5Ka0PCgqCqamp0n4uKSmBWCzulPu5pXybcu7cOQDo1Pv5UXK5HPX19brdv7odW9T1fP3118zc3Jxt2bKFXb58mc2bN4/Z29uzyspKfYfWLl577TWWn5/PysrK2PHjx1lERATr2bMnu3Xrlr5D05mamhp29uxZdvbsWQaAffTRR+zs2bPsjz/+YIwxtnLlSmZvb8++/fZbduHCBRYTE8O8vLzYvXv39By5dprLt6amhi1YsIAVFRWxsrIydujQIRYYGMi8vb3Z/fv39R261l588UVmZ2fH8vPzWUVFBfdTV1fHtZk/fz4TCoXs8OHD7MyZMyw4OJgFBwfrMWrttZTv1atX2fLly9mZM2dYWVkZ+/bbb1mfPn1YaGioniPX3qJFi9jRo0dZWVkZu3DhAlu0aBHj8Xjs4MGDjDHd7V8qkq2wdu1aJhQKmZmZGRs+fDg7efKkvkNqN8888wxzdXVlZmZmrHfv3uyZZ55hV69e1XdYOnXkyBEGQOVnxowZjLEH00CWLFnCnJ2dmbm5ORszZgwrKSnRb9Bt0Fy+dXV1bNy4cczR0ZGZmpoyDw8PNnfu3E7/R2BT+QJgmzdv5trcu3ePvfTSS6x79+7MysqKTZkyhVVUVOgv6DZoKV+xWMxCQ0OZg4MDMzc3Z/369WMLFy5k1dXV+g28DWbNmsU8PDyYmZkZc3R0ZGPGjOEKJGO627/0qCxCCCFEDbomSQghhKhBRZIQQghRg4okIYQQogYVSUIIIUQNKpKEEEKIGlQkCSGEEDWoSBJCCCFqUJEkhBBC1KAiSUgHSU9Px5AhQzR6D4/Hw969e9slHl0oLy8Hj8fj7gNKSFdDRZIQLfB4vGZ/0tPTVd6zYMECpRsu60JiYiJ4PB5WrlyptHzv3r3g8Xg6/SxCjBE9T5IQLVRUVHD/3rFjB95++22UlJRwy2xsbLh/M8Ygk8lgY2OjtFxXLCwssGrVKrzwwgvo3r27zrevDw0NDTAzM9N3GITQmSQh2nBxceF+7OzswOPxuNe//vorbG1tsX//fgQFBcHc3ByFhYUq3a2nT5/G2LFj0bNnT9jZ2WHUqFH4+eefNY4lIiICLi4uWLFihdo2TXX1ZmRkwNPTk3udmJiIyZMn4/3334ezszPs7e2xfPlyNDY2YuHChXBwcICbmxs2b96ssv1ff/0VTzzxBCwsLDBo0CCVB99eunQJ48ePh42NDZydnfHcc8/hr7/+4taHhYUhOTkZqamp6NmzJyIjIzX+HghpD1QkCWknixYtwsqVKyESiRAQEKCyvqamBjNmzEBhYSFOnjwJb29vTJgwATU1NRp9jomJCd5//32sXbsWf/75Z5tiPnz4MG7cuIFjx47ho48+wtKlSzFp0iR0794dp06dwvz58/HCCy+ofM7ChQvx2muv4ezZswgODkZ0dDSqqqoAAHfu3MHo0aPx2GOP4cyZM8jJycHNmzcRHx+vtI2tW7fCzMwMx48fx8aNG9uUByG6QkWSkHayfPlyjB07Fn379oWDg4PK+tGjR+Nf//oXfH194efnh//973+oq6tTOQtrjSlTpmDIkCFYunRpm2J2cHDAmjVr4OPjg1mzZsHHxwd1dXV488034e3tjcWLF8PMzAyFhYVK70tOTkZcXBz8/PywYcMG2NnZYdOmTQCATz75BI899hjef/99+Pr64rHHHsPnn3+OI0eO4LfffuO24e3tjdWrV8PHxwc+Pj5tyoMQXaEiSUg7GTp0aLPrb968iblz58Lb2xt2dnYQCASora2FWCzW6vNWrVqFrVu3QiQSafV+ABg4cCD4/P/7teDs7Ax/f3/utYmJCXr06IFbt24pve/hp71369YNQ4cO5eI4f/48jhw5wl2TtbGxga+vLwCgtLSUe19QUJDWcRPSXmjgDiHtxNrautn1M2bMQFVVFTIzM+Hh4QFzc3MEBwejoaFBq88LDQ1FZGQkFi9ejMTERKV1fD4fjz46ViqVqmzD1NRU6TWPx2tymVwub3VctbW1iI6OxqpVq1TWubq6cv9u6fsiRB+oSBKiJ8ePH8f69esxYcIEAMC1a9eUBrNoY+XKlRgyZIhKd6WjoyMqKyvBGOOmhuhybuPJkycRGhoKAGhsbERxcTGSk5MBAIGBgfjmm2/g6emJbt3oVw7pXKi7lRA98fb2xrZt2yASiXDq1ClMnz4dlpaWbdqmv78/pk+fjjVr1igtDwsLw+3bt7F69WqUlpZi3bp12L9/f5s+62Hr1q3Dnj178OuvvyIpKQn//PMPZs2aBQBISkrC33//jWeffRanT59GaWkpDhw4gJkzZ0Imk+ksBkLaAxVJQvRk06ZN+OeffxAYGIjnnnsOr7zyCpycnNq83eXLl6t0h/r5+WH9+vVYt24dBg8ejJ9++gkLFixo82cprFy5EitXrsTgwYNRWFiI7777Dj179gQA9OrVC8ePH4dMJsO4cePg7++P1NRU2NvbK13/JMQQ8dijFyoIIYQQAoDOJAkhhBC1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtT4f/vMJmwbuC7rAAAAAElFTkSuQmCC",
+ "text/plain": [
+ "<Figure size 500x300 with 1 Axes>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "trial_logs = compiled_program.trial_logs\n",
+ "\n",
+ "# Extracting trial numbers, scores, and pruning status\n",
+ "trial_numbers = list(trial_logs.keys())\n",
+ "scores = [trial_logs[trial]['score'] for trial in trial_numbers]\n",
+ "pruning_status = [trial_logs[trial]['pruned'] for trial in trial_numbers]\n",
+ "\n",
+ "# Plot setup\n",
+ "plt.figure(figsize=(5, 3))\n",
+ "\n",
+ "# Plotting each point\n",
+ "for trial_number, score, pruned in zip(trial_numbers, scores, pruning_status):\n",
+ " if pruned:\n",
+ " plt.scatter(trial_number, score, color='grey', label='Pruned Trial' if 'Pruned Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",
+ " else:\n",
+ " plt.scatter(trial_number, score, color='green', label='Successful Trial' if 'Successful Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",
+ "\n",
+ "plt.xlabel('Trial Number')\n",
+ "plt.ylabel('Score')\n",
+ "plt.title('Trial Scores with Pruning Status')\n",
+ "plt.grid(True)\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can also visualize the best prompts discovered by MIPRO as our trials progress... "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Basline program | Score: 0:\n",
+ "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",
+ "Prompt 2 Instruction: Given the fields `context`, `question`, produce the fields `answer`.\n",
+ "Prompt 3 Instruction: Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\n",
+ "\n",
+ "----------------\n",
+ "Best program after 0 trials | Score: 24.0:\n",
+ "Prompt 1 Instruction: Given the fact-based nature of the questions related to pop culture, history, and entertainment, with a focus on identifying specific works or individuals, accurately retrieve and synthesize relevant information.\n",
+ "Prompt 2 Instruction: Given the fields `context` containing information about a specific topic, and `question` containing a fact-based question, generate the field `answer` with the specific information that directly addresses the question.\n",
+ "Prompt 3 Instruction: For a given question and list of passages, rank the passages in order of helpfulness in answering the question, with the most helpful passage first and the least helpful last. Provide a comma-separated list of the passage numbers in the ranked order.\n",
+ "\n",
+ "Best program after 5 trials | Score: 31.0:\n",
+ "Prompt 1 Instruction: Given the fact-based nature of the questions related to pop culture, history, and entertainment, with a focus on identifying specific works or individuals, accurately retrieve and synthesize relevant information.\n",
+ "Prompt 2 Instruction: Given the fields `context`, `question`, produce the fields `answer`.\n",
+ "Prompt 3 Instruction: Given a fact-based question, identify the specific work, individual, or event being referenced, and provide a concise and accurate answer based on the information provided in the passages.\n",
+ "\n",
+ "Best program after 10 trials | Score: 41.4:\n",
+ "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",
+ "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment, along with a corresponding context, identify and provide a concise answer that directly corresponds to the specific question posed.\n",
+ "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",
+ "\n",
+ "Best program after 15 trials | Score: 42.4:\n",
+ "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",
+ "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",
+ "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",
+ "\n",
+ "Best program after 20 trials | Score: 42.4:\n",
+ "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",
+ "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",
+ "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",
+ "\n",
+ "Best program after 25 trials | Score: 42.4:\n",
+ "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",
+ "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",
+ "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "best_score = 0\n",
+ "\n",
+ "def get_signature(predictor):\n",
+ " if (hasattr(predictor, 'extended_signature')):\n",
+ " return predictor.extended_signature\n",
+ " elif (hasattr(predictor, 'signature')):\n",
+ " return predictor.signature\n",
+ "\n",
+ "print(f\"Basline program | Score: {best_score}:\")\n",
+ "for i,predictor in enumerate(program.predictors()):\n",
+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",
+ "print() \n",
+ "\n",
+ "print(\"----------------\")\n",
+ "\n",
+ "for trial_num in compiled_program.trial_logs:\n",
+ " program_score = compiled_program.trial_logs[trial_num][\"score\"]\n",
+ " program_pruned = compiled_program.trial_logs[trial_num][\"pruned\"]\n",
+ " if program_score > best_score and not program_pruned:\n",
+ " best_score = program_score\n",
+ " best_program_so_far = compiled_program.trial_logs[trial_num][\"program\"]\n",
+ " if trial_num % 5 == 0:\n",
+ " print(f\"Best program after {trial_num} trials | Score: {best_score}:\")\n",
+ " for i,predictor in enumerate(best_program_so_far.predictors()):\n",
+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",
+ " print() "
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "dspy_test",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}