DSPy: The framework for programming—not prompting—language models
Python
MIPRO optimizer updates for paper release
Michael Ryan38371f86c209
diff --git a/.gitignore b/.gitignoreindex 45557810a..2baf0d990 100644--- a/.gitignore+++ b/.gitignore@@ -8,6 +8,10 @@/docs/downloads//docs/experiments/+/examples/qa/hotpot/MIPRO_notebook_cache/+/examples/nli/scone/MIPRO_notebook_cache/+/ScoNe/+# Byte-compiled / optimized / DLL files__pycache__/*.py[cod]diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.pyindex 3ee615574..6f442f643 100644--- a/dsp/primitives/predict.py+++ b/dsp/primitives/predict.py@@ -1,11 +1,12 @@from collections import Counter-from typing import Any, Callable, Optional+from typing import Callable, Any, Optionalimport dsp-from dsp.primitives.demonstrate import Example-from dsp.templates.template_v3 import Template-from dsp.utils import normalize_text, zipstar+from dsp.utils import zipstar, normalize_text+from dsp.primitives.inspect import FuncInspectorfrom dsp.utils.utils import dotdict+from dsp.templates.template_v3 import Template+from dsp.primitives.demonstrate import Exampleclass Completions:@@ -61,43 +62,8 @@ def _generate(template: Template, **kwargs) -> Callable:generator = dsp.settings.lm- def extend_generation(completion: Example, field_names: list[str], stage:str, max_depth: int, original_example:Example):- """If the required fields are not present in the completion, extend the generation."""- assert max_depth > 0, "Max depth exceeded - failed to complete in one pass - increase max_tokens"- # remove content of last field to avoid half-completed content- for field_name in get_all_fields_following_missing_field(completion, field_names):- completion.pop(field_name, None)-- # Recurse with greedy decoding and a shorter length.- max_tokens = (kwargs.get("max_tokens") or- kwargs.get("max_output_tokens") or- dsp.settings.lm.kwargs.get("max_tokens") or- dsp.settings.lm.kwargs.get('max_output_tokens'))--- if max_tokens is None:- raise ValueError("Required 'max_tokens' or 'max_output_tokens' not specified in settings.")- max_tokens = min(max(75, max_tokens // 2), max_tokens)- keys = list(kwargs.keys()) + list(dsp.settings.lm.kwargs.keys())- max_tokens_key = "max_tokens" if "max_tokens" in keys else "max_output_tokens"- new_kwargs = {- **kwargs,- max_tokens_key: max_tokens,- "n": 1,- "temperature": 0.0,- }-- _, finished_completion = generate(template, **new_kwargs)(- completion,- stage=stage,- max_depth=max_depth - 1,- original_example=original_example,- )- return finished_completion.data[0]--def do_generate(- example: Example, stage: str, max_depth: int = 2, original_example=None,+ example: Example, stage: str, max_depth: int = 2, original_example=None):if not dsp.settings.lm:raise AssertionError("No LM is loaded.")@@ -112,19 +78,45 @@ def _generate(template: Template, **kwargs) -> Callable:completions: list[dict[str, Any]] = generator(prompt, **kwargs)completions: list[Example] = [template.extract(example, p) for p in completions]- # Find the completions that are unfinished.+ # Find the completions that are most complete.field_names: list[str] = [field.input_variable for field in template.fields]- finished_completions = []- for completion in completions:- if all((completion.get(key, "") != "") for key in field_names):- finished_completions.append(completion)- continue- finished_completions.append(- extend_generation(completion, field_names, stage, max_depth, original_example),+ last_field_idx = 0+ for field_idx, key in enumerate(field_names):+ completions_ = [+ c for c in completions if key in c.keys() and c[key] is not None+ ]++ # Filter out completions that are missing fields that are present in at least one completion.+ if len(completions_):+ completions = completions_+ last_field_idx = field_idx + 1++ # If none of the completions is completed (i.e., none has the final field set).+ if last_field_idx < len(field_names):+ # Pick the first completion that has gone farthest.+ completion = completions[0]+ completion[field_names[last_field_idx]] = ""++ # Recurse with greedy decoding and a shorter length.+ max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"])+ max_tokens = min(max(75, max_tokens // 2), max_tokens)+ new_kwargs = {+ **kwargs,+ "max_tokens": max_tokens,+ "n": 1,+ "temperature": 0.0,+ }++ assert max_depth > 0+ return generate(template, **new_kwargs)(+ completion,+ stage=stage,+ max_depth=max_depth - 1,+ original_example=original_example,)- completions = Completions(finished_completions, template=template)+ completions = Completions(completions, template=template)example = example.copy(completions=completions)if len(completions) == 1:@@ -151,7 +143,7 @@ def _generate(template: Template, **kwargs) -> Callable:"template": template,"inputs": inputs,"outputs": outputs,- },+ })else:# assert not dsp.settings.compiling, "TODO: At this point, cannot compile n>1 generations"@@ -161,18 +153,9 @@ def _generate(template: Template, **kwargs) -> Callable:return do_generate-def get_all_fields_following_missing_field(completion: Example, field_names: list[str]) -> list[str]:- """Returns every field following the first missing field"""- for i, field_name in enumerate(field_names):- if field_name not in completion:- return field_names[i:]- if completion[field_name] == "":- return field_names[i:]- return []-def generate_sc(- example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs,+ example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs):if not dsp.settings.lm:raise AssertionError("No LM is loaded.")@@ -181,7 +164,7 @@ def generate_sc(completions = dsp.settings.lm(prompt, **kwargs)completions = extract_final_answer(example, completions, extract=extract)return majority_vote_(- completions, normalize=normalize, prediction_field=prediction_field,+ completions, normalize=normalize, prediction_field=prediction_field)@@ -197,14 +180,14 @@ def extract_final_answer(example, completions, extract=None):# TODO: make thread-safe?dsp.settings.lm.history.append(- {**dsp.settings.lm.history[-1], "completions": completions},+ {**dsp.settings.lm.history[-1], "completions": completions})return completionsdef majority(- completions: Completions, normalize: bool = True, field: Optional[str] = None,+ completions: Completions, normalize: bool = True, field: Optional[str] = None):"""Returns the most common completion for the target field or the last field in the template."""field = completions.template.fields[-1].output_variable if field is None else field@@ -248,7 +231,7 @@ def majority_vote_(completions: Completions, normalize: bool, prediction_field:pred = normalized_to_original[pred]dsp.settings.lm.history.append(- {**dsp.settings.lm.history[-1], "topk": topk, "completions": [pred]},+ {**dsp.settings.lm.history[-1], "topk": topk, "completions": [pred]})return [pred]diff --git a/dspy/propose/__init__.py b/dspy/propose/__init__.pynew file mode 100644index 000000000..37d3cacd4--- /dev/null+++ b/dspy/propose/__init__.py@@ -0,0 +1 @@+from .grounded_proposer import GroundedProposer\ No newline at end of filediff --git a/dspy/propose/dataset_summary_generator.py b/dspy/propose/dataset_summary_generator.pynew file mode 100644index 000000000..484f4a99b--- /dev/null+++ b/dspy/propose/dataset_summary_generator.py@@ -0,0 +1,84 @@+import dspy+import re+from dspy.propose.utils import strip_prefix++class ObservationSummarizer(dspy.Signature):+ ("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")+ observations = dspy.InputField(desc="Observations I have made about my dataset")+ summary = dspy.OutputField(desc="Two to Three sentence summary of only the most significant highlights of my observations")++class DatasetDescriptor(dspy.Signature):+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")++ examples = dspy.InputField(desc="Sample data points from the dataset")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")++class DatasetDescriptorWithPriorObservations(dspy.Signature):+ ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")++ examples = dspy.InputField(desc="Sample data points from the dataset")+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")++def order_input_keys_in_string(unordered_repr):+ # Regex pattern to match the input keys structure+ pattern = r"input_keys=\{([^\}]+)\}"++ # Function to reorder keys+ def reorder_keys(match):+ # Extracting the keys from the match+ keys_str = match.group(1)+ # Splitting the keys, stripping extra spaces, and sorting them+ keys = sorted(key.strip() for key in keys_str.split(','))+ # Formatting the sorted keys back into the expected structure+ return f"input_keys={{{', '.join(keys)}}}"++ # Using re.sub to find all matches of the pattern and replace them using the reorder_keys function+ ordered_repr = re.sub(pattern, reorder_keys, unordered_repr)++ return ordered_repr++def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None):+ upper_lim = min(len(trainset), view_data_batch_size)+ with dspy.settings.context(lm=prompt_model):+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=order_input_keys_in_string(trainset[0:upper_lim].__repr__()))+ observations = observation["observations"]++ if log_file:+ log_file.write("PRODUCING DATASET SUMMARY\n")++ skips = 0+ try:+ max_calls = 10+ calls = 0+ for b in range(view_data_batch_size, len(trainset), view_data_batch_size):+ calls+=1+ if calls >= max_calls:+ break+ print(f"b: {b}")+ upper_lim = min(len(trainset), b+view_data_batch_size)+ with dspy.settings.context(lm=prompt_model):+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=order_input_keys_in_string(trainset[b:upper_lim].__repr__()))+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":+ skips += 1+ if skips >= 5:+ break+ continue+ observations += output["observations"]++ log_file.write(f"observations {observations}\n")+ except Exception as e:+ print(f"e {e}. using observations from past round for a summary.")++ with dspy.settings.context(lm=prompt_model):+ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)+ print(f"summary: {summary}")+ if log_file:+ log_file.write(f"summary: {summary}\n")++ return strip_prefix(summary.summary)\ No newline at end of filediff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.pynew file mode 100644index 000000000..f3df1de15--- /dev/null+++ b/dspy/propose/grounded_proposer.py@@ -0,0 +1,371 @@+from .propose_base import Proposer+import dspy+import re+from dspy.teleprompt.utils import get_signature+from dspy.propose.utils import strip_prefix, create_example_string, create_predictor_level_history_string+from dspy.propose.dataset_summary_generator import create_dataset_summary+import random++# Hardcoded variables (TODO: update)+MAX_INSTRUCT_IN_HISTORY = 5 # 10++TIPS = {+ "none": "",+ "creative": "Don't be afraid to be creative when creating the new instruction!",+ "simple": "Keep the instruction clear and concise.",+ "description": "Make sure your instruction is very informative and descriptive.",+ "high_stakes": "The instruction should include a high stakes scenario in which the LM must solve the task!",+ "persona": 'Include a persona that is relevant to the task in the instruction (ie. "You are a ...")',+ }++### SIGNATURES USED TO HELP WITH INSTRUCTION GENERATION ###++class DescribeProgram(dspy.Signature):+ (+ """Below is some pseudo-code for a pipeline that solves tasks with calls to language models. Please describe what type of task this program appears to be designed to solve, and how it appears to work."""+ )+ program_code = dspy.InputField(+ format=str,+ desc="Pseudocode for a language model program designed to solve a particular task.",+ prefix="PROGRAM CODE:",+ )+ program_example = dspy.InputField(+ format=str,+ desc="An example of the program in use.",+ prefix="EXAMPLE OF PROGRAM IN USE:",+ )+ program_description = dspy.OutputField(+ desc="Describe what task the program is designed to solve, and how it goes about solving this task.",+ prefix="SUMMARY OF PROGRAM ABOVE:",+ )+++class DescribeModule(dspy.Signature):+ (+ """Below is some pseudo-code for a pipeline that solves tasks with calls to language models. Please describe the purpose of one of the specified module in this pipeline."""+ )+ program_code = dspy.InputField(+ format=str,+ desc="Pseudocode for a language model program designed to solve a particular task.",+ prefix="PROGRAM CODE:",+ )+ program_example = dspy.InputField(+ format=str,+ desc="An example of the program in use.",+ prefix="EXAMPLE OF PROGRAM IN USE:",+ )+ program_description = dspy.InputField(+ desc="Summary of the task the program is designed to solve, and how it goes about solving it.",+ prefix="SUMMARY OF PROGRAM ABOVE:",+ )+ module = dspy.InputField(+ desc="The module in the program that we want to describe.", prefix="MODULE:"+ )+ module_description = dspy.OutputField(+ desc="Description of the module's role in the broader program.",+ prefix="MODULE DESCRIPTION:",+ )+++def generate_instruction_class(+ use_dataset_summary=True,+ program_aware=True,+ use_task_demos=True,+ use_instruct_history=True,+ use_tip=True,+):+ class GenerateSingleModuleInstruction(dspy.Signature):+ (+ """Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task."""+ )+ if use_dataset_summary:+ dataset_description = dspy.InputField(+ desc="A description of the dataset that we are using.",+ prefix="DATASET SUMMARY:",+ )+ if program_aware:+ program_code = dspy.InputField(+ format=str,+ desc="Language model program designed to solve a particular task.",+ prefix="PROGRAM CODE:",+ )+ program_description = dspy.InputField(+ desc="Summary of the task the program is designed to solve, and how it goes about solving it.",+ prefix="PROGRAM DESCRIPTION:",+ )+ module = dspy.InputField(+ desc="The module to create an instruction for.", prefix="MODULE:"+ )+ task_demos = dspy.InputField(+ format=str,+ desc="Example inputs/outputs of our module.",+ prefix="TASK DEMO(S):",+ )+ if use_instruct_history:+ previous_instructions = dspy.InputField(+ format=str,+ desc="Previous instructions we've attempted, along with their associated scores.",+ prefix="PREVIOUS INSTRUCTIONS:",+ )+ basic_instruction = dspy.InputField(+ format=str, desc="Basic instruction.", prefix="BASIC INSTRUCTION:"+ )+ if use_tip:+ tip = dspy.InputField(+ format=str,+ desc="A suggestion for how to go about generating the new instruction.",+ prefix="TIP:",+ )+ proposed_instruction = dspy.OutputField(+ desc="Propose an instruction that will be used to prompt a Language Model to perform this task.",+ prefix="PROPOSED INSTRUCTION:",+ )++ return dspy.Predict(GenerateSingleModuleInstruction)++### CLASS RESPONSIBLE FOR GENERATING A NEW INSTRUCTION, USING THE HELPER SIGNATURES ABOVE ###++class GenerateModuleInstruction(dspy.Module):+ def __init__(+ self,+ program_code_string=None,+ use_dataset_summary=True,+ program_aware=False,+ use_task_demos=True,+ use_instruct_history=True,+ use_tip=True,+ ):+ super().__init__()+ self.use_dataset_summary = use_dataset_summary+ self.program_aware = program_aware+ self.use_task_demos = use_task_demos+ self.use_instruct_history = use_instruct_history+ self.use_tip = use_tip++ self.program_code_string = program_code_string+ self.describe_program = dspy.Predict(DescribeProgram)+ self.describe_module = dspy.Predict(DescribeModule)+ self.generate_module_instruction = generate_instruction_class(+ use_dataset_summary=use_dataset_summary,+ program_aware=program_aware,+ use_task_demos=use_task_demos,+ use_instruct_history=use_instruct_history,+ use_tip=use_tip,+ )++ def forward(+ self,+ demo_candidates,+ pred_i,+ demo_set_i,+ program,+ previous_instructions,+ data_summary,+ max_demos=3,+ tip=None,+ ):+ # Construct full program demo or single module demo depending on whether or not we're using the full program+ task_demos = ""+ basic_instruction = get_signature(program.predictors()[pred_i]).instructions+ curr_demos_num = 0+ # if self.program_aware: # Use full program demo+ # for pred_key in demo_candidates.keys(): # Go through each predictor+ # for example in demo_candidates[pred_key][demo_set_i]:+ # if "augmented" in example.keys():+ # fields_to_use = get_signature(program.predictors()[pred_key]).fields+ # example_string = create_example_string(fields_to_use, example)+ # task_demos += f"{example_string}\n"+ # curr_demos_num+=1+ # if curr_demos_num>=max_demos:+ # break+ # else:+ for example in demo_candidates[pred_i][demo_set_i]:+ if "augmented" in example.keys():+ fields_to_use = get_signature(program.predictors()[pred_i]).fields+ example_string = create_example_string(fields_to_use, example)+ task_demos += f"{example_string}\n"+ curr_demos_num += 1+ if curr_demos_num >= max_demos:+ break++ # Summarize the program+ program_description = ""+ module_code = ""+ if self.program_aware:+ program_description = strip_prefix(+ self.describe_program(+ program_code=self.program_code_string, program_example=task_demos+ ).program_description+ )+ print(f"PROGRAM DESCRIPTION: {program_description}")++ # Identify all modules+ init_pattern = r"def __init__.*?\):([\s\S]*?)(?=\n\s{4}def|\Z)"+ init_content_match = re.search(init_pattern, self.program_code_string)+ init_content = init_content_match.group(0)+ pattern = r"^(.*dspy\.(ChainOfThought|Predict).*)$" # TODO: make it so that this extends out to any dspy Module+ matches = re.findall(pattern, init_content, re.MULTILINE)+ modules = [match[0].strip() for match in matches]+ module_code = modules[pred_i]++ module_description = self.describe_module(+ program_code=self.program_code_string,+ program_description=program_description,+ program_example=task_demos,+ module=module_code,+ max_depth=10,+ ).module_description++ # Generate an instruction for our chosen module+ print(f"task_demos {task_demos}")+ instruct = self.generate_module_instruction(+ dataset_description=data_summary,+ program_code=self.program_code_string,+ program_description=program_description,+ module=module_code,+ task_demos=task_demos,+ tip=tip,+ basic_instruction=basic_instruction,+ previous_instructions=previous_instructions,+ module_description=module_description,+ )+ if hasattr(instruct, "module_description"):+ module_description = strip_prefix(instruct.module_description)+ print(f"MODULE DESCRIPTION: {module_description}")+ proposed_instruction = strip_prefix(instruct.proposed_instruction)+ # print(f"PROPOSED INSTRUCTION: {proposed_instruction}")++ return dspy.Prediction(proposed_instruction=proposed_instruction)++### CLASS USED TO GENERATE THE FULL SET OF INSTRUCTIONS GIVEN THE SPECIFIED CRITERIA ###++class GroundedProposer(Proposer):+ def __init__(+ self,+ prompt_model,+ trainset,+ program_code_string=None,+ use_dataset_summary=True,+ program_aware=True,+ use_task_demos=True,+ use_instruct_history=True,+ use_tip=True,+ set_tip_randomly=True,+ set_history_randomly=True,+ ):+ super().__init__()+ self.program_aware = program_aware+ self.use_dataset_summary = use_dataset_summary+ self.use_task_demos = use_task_demos+ self.use_instruct_history = use_instruct_history+ self.use_tip = use_tip+ self.set_tip_randomly=set_tip_randomly+ self.set_history_randomly=set_history_randomly++ self.prompt_model = prompt_model+ self.program_code_string = program_code_string+ self.data_summary = create_dataset_summary(+ trainset=trainset, view_data_batch_size=10, prompt_model=prompt_model+ )+ print(f"DATA SUMMARY: {self.data_summary}")++ def propose_instructions_for_program(+ self,+ trainset,+ program,+ demo_candidates,+ prompt_model,+ trial_logs,+ N,+ T,+ tip=None,+ ):+ """This method is responsible for returning the full set of new instructions for our program, given the specified criteria."""++ proposed_instructions = {}++ if self.set_tip_randomly:+ print(f"Using a randomly generated configuration for our grounded proposer.")+ # Randomly select the tip+ selected_tip_key = random.choice(list(TIPS.keys()))+ selected_tip = TIPS[selected_tip_key]+ self.use_tip = bool(+ selected_tip+ )+ print(f"Selected tip: {selected_tip_key}")++ if self.set_history_randomly:+ # Randomly select whether or not we're using instruction history+ use_history = random.random() < 0.5+ self.use_instruct_history = use_history+ print(f"Use history T/F: {self.use_instruct_history}")++ # Create an instruction for each predictor+ for pred_i, predictor in enumerate(program.predictors()):+ for demo_set_i in range(len(demo_candidates[0])):+ if pred_i not in proposed_instructions:+ proposed_instructions[pred_i] = []+ proposed_instructions[pred_i].append(+ self.propose_instruction_for_predictor(+ program=program,+ predictor=predictor,+ pred_i=pred_i,+ prompt_model=prompt_model,+ T=T,+ demo_candidates=demo_candidates,+ demo_set_i=demo_set_i,+ trial_logs=trial_logs,+ tip=selected_tip,+ )+ )+ return proposed_instructions++ def propose_instruction_for_predictor(+ self,+ program,+ predictor,+ pred_i,+ prompt_model,+ T,+ demo_candidates,+ demo_set_i,+ trial_logs,+ tip=None,+ ):+ """This method is responsible for returning a single instruction for a given predictor, using the specified criteria."""++ # Create an instruction history string for our predictor+ instruction_history = create_predictor_level_history_string(+ program, pred_i, trial_logs, MAX_INSTRUCT_IN_HISTORY+ )++ # Create our instruction generator class (given specific criteria for this round of proposal)+ instruction_generator = GenerateModuleInstruction(+ program_code_string=self.program_code_string,+ use_dataset_summary=self.use_dataset_summary,+ program_aware=self.program_aware,+ use_task_demos=self.use_task_demos,+ use_instruct_history=self.use_instruct_history and instruction_history,+ use_tip=self.use_tip,+ )++ # Generate a new instruction for our predictor, using the temperature specified for this round+ original_temp = prompt_model.kwargs["temperature"]+ with dspy.settings.context(lm=prompt_model):+ prompt_model.kwargs["temperature"] = T+ proposed_instruction = instruction_generator.forward(+ demo_candidates=demo_candidates,+ pred_i=pred_i,+ demo_set_i=demo_set_i,+ program=program,+ data_summary=self.data_summary,+ previous_instructions=instruction_history,+ tip=tip,+ ).proposed_instruction+ prompt_model.kwargs["temperature"] = original_temp++ # Log the trace used to generate the new instruction, along with the new instruction itself+ prompt_model.inspect_history(n=1)+ print(f"PROPOSED INSTRUCTION: {proposed_instruction}")++ return strip_prefix(proposed_instruction)diff --git a/dspy/propose/instruction_proposal.py b/dspy/propose/instruction_proposal.pynew file mode 100644index 000000000..5f3b48483--- /dev/null+++ b/dspy/propose/instruction_proposal.py@@ -0,0 +1,129 @@+from dspy.propose.dataset_summary_generator import observe_data+from dspy.teleprompt.utils import get_signature+from dspy.signatures import Signature+import dsp+import dspy+from collections import defaultdict++class BasicGenerateInstruction(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will provide you with one or more ``example instruction(s)`` that were previously attempted for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ example_instructions = dspy.InputField(format=dsp.passages2text, desc="Example instruction(s) for this task.")+ proposed_instruction = dspy.InputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionWithExamplesAndDataObservationsAndTip(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you a summary I have made about the dataset, along with some ``examples`` of the expected inputs and outputs for this task. I will also provide you with one or more ``example instruction(s)`` that were previously attempted for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ dataset_summary = dspy.InputField(desc="Summary of the dataset.")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ example_instructions = dspy.InputField(format=dsp.passages2text, desc="Example instruction(s) for this task.")+ tip = dspy.InputField(desc="A tip for something to keep in mind when generating the new instruction.")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionWithDataObservationsAndTip(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you a summary I have made about the dataset, along with some ``examples`` of the expected inputs and outputs for this task. I will also provide you with one or more ``example instruction(s)`` that were previously attempted for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ dataset_summary = dspy.InputField(desc="Summary of the dataset.")+ example_instructions = dspy.InputField(format=dsp.passages2text, desc="Example instruction(s) for this task.")+ tip = dspy.InputField(desc="A tip for something to keep in mind when generating the new instruction.")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionWithExamplesAndTip(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``examples`` of the expected inputs and outputs for this task. I will also provide you with one or more ``example instruction(s)`` that were previously attempted for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ example_instructions = dspy.InputField(format=dsp.passages2text, desc="Example instruction(s) for this task.")+ tip = dspy.InputField(desc="A tip for something to keep in mind when generating the new instruction.")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionWithTip(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will provide you with one or more ``example instruction(s)`` that were previously attempted for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ example_instructions = dspy.InputField(format=dsp.passages2text, desc="Example instruction(s) for this task.")+ tip = dspy.InputField(desc="A tip for something to keep in mind when generating the new instruction.")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")+++class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.++Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ observations = dspy.InputField(desc="Observations about the dataset and task")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionWithExamples(dspy.Signature):+ ("""You are an instruction optimizer for large language models. I will also give you one or more ``examples`` of the expected inputs and outputs.++Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative.""")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")+++class BasicGenerateInstructionWithDataObservations(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ observations = dspy.InputField(desc="Observations about the dataset and task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")+++class BasicGenerateInstruction(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionAllFields(Signature):+ """You are an instruction optimizer for large language models. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""+ ("""You are an instruction optimizer for large language models. I will provide you with""")+ ("""the current instruction that I am using, along with a template consisting of names""")+ ("""and prefixes for the different input and output fields for the model. Your task is to propose a new instruction""")+ ("""and template for these input and output fields that will help the language model perform the task well. Don't be afraid to be creative.""")++ initial_prompt_template = dspy.InputField(desc="The initial prompt template for this task.")+ proposed_prompt_template = dspy.OutputField(desc="The improved prompt template for this task.")+ # proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class BasicGenerateInstructionOnly(Signature):+ ("""You are an instruction optimizer for large language models. I will provide you with an instruction I'm currently using. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative.""")++ current_instruction = dspy.InputField(desc="The current instruction.")+ # examples_of_field_in_use = dspy.InputField(format=dsp.passages2text, desc="Examples of this field in use on examples from our task.")+ proposed_instruction = dspy.OutputField(desc="The proposed instruction (reply with the instruction only).")++class BasicGenerateField(Signature):+ ("""You are an instruction optimizer for large language models. Your task is to propose a better string to use for one of the fields+ in a prompt that is being inputted to a large language model to perform a certain task. The goal is for this improved field to improve+ the performance of the language model on this task. Don't be afraid to be creative.""")++ current_field = dspy.InputField(desc="The current string in use for this field.")+ examples_of_field_in_use = dspy.InputField(format=dsp.passages2text, desc="Examples of this field in use on examples from our task.")+ proposed_field = dspy.OutputField(desc="The proposed string for the field (respond with the new field string only).")+ # proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")++class GenerateInstructionGivenAttempts(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give some task instructions I've tried, along with the corresponding validation scores, where higher scores indicate better quality. I will also include an example of each instruction in use on a randomly chosen sample from our validation set.++Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative."""++ attempted_instructions = dspy.InputField(format=dsp.passages2text)+ # attempted_instructions = dspy.InputField(desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")\ No newline at end of filediff --git a/dspy/propose/propose_base.py b/dspy/propose/propose_base.pynew file mode 100644index 000000000..db1ec8132--- /dev/null+++ b/dspy/propose/propose_base.py@@ -0,0 +1,13 @@+from abc import ABC, abstractmethod+++class Proposer(ABC):+ def __init__(self):+ pass++ @abstractmethod+ def propose_instructions_for_program(self):+ pass++ def propose_instruction_for_predictor(self):+ pass\ No newline at end of filediff --git a/dspy/propose/utils.py b/dspy/propose/utils.pynew file mode 100644index 000000000..7d70e26e4--- /dev/null+++ b/dspy/propose/utils.py@@ -0,0 +1,198 @@+import re+import json+from dspy.teleprompt.utils import get_signature++def strip_prefix(text):+ pattern = r'^[\*\s]*(([\w\'\-]+\s+){0,4}[\w\'\-]+):\s*'+ modified_text = re.sub(pattern, '', text)+ return modified_text.strip("\"")++def create_instruction_set_history_string(base_program, trial_logs, top_n):+ program_history = []+ for trial_num in trial_logs:+ trial = trial_logs[trial_num]+ if "program_path" in trial:+ trial_program = base_program.deepcopy()+ trial_program.load(trial["program_path"])+ program_history.append({+ "program": trial_program,+ "score": trial["score"]+ })++ # Deduplicate program history based on the program's instruction set+ seen_programs = set()+ unique_program_history = []+ for entry in program_history:+ program = entry["program"]+ instruction_set = get_program_instruction_set_string(program)+ if instruction_set not in seen_programs:+ seen_programs.add(instruction_set)+ unique_program_history.append(entry)++ # Get the top n programs from program history+ top_n_program_history = sorted(unique_program_history, key=lambda x: x['score'], reverse=True)[:top_n]+ top_n_program_history.reverse()++ # Create formatted string+ instruction_set_history_string = ""+ for entry in top_n_program_history:+ program = entry["program"]+ score = entry["score"]+ instruction_set = get_program_instruction_set_string(program)+ instruction_set_history_string += instruction_set + f" | Score: {score}\n\n"++ return instruction_set_history_string++# def parse_list_of_instructions(instruction_string):+# # Convert the string representation of a list to an actual list+# try:+# instructions = json.loads(instruction_string)+# except json.JSONDecodeError as e:+# print(f"Error decoding JSON: {e}")+# return []++# return instructions+def parse_list_of_instructions(instruction_string):+ # Try to convert the string representation of a list to an actual list using JSON+ try:+ instructions = json.loads(instruction_string)+ return instructions+ except json.JSONDecodeError:+ pass++ # If JSON decoding fails, extract strings within quotes+ instructions = re.findall(r'"([^"]*)"', instruction_string)+ return instructions++# def get_program_instruction_set_string(program):+# instruction_set_dict = {}+# for i, pred in enumerate(program.predictors()):+# pred_instructions = get_signature(pred).instructions+# instruction_set_dict[f"Module {i} Instruction"] = pred_instructions+# return '\n'.join(f"{key}: {value}" for key, value in instruction_set_dict.items())++def get_program_instruction_set_string(program):+ instruction_list = []+ for _, pred in enumerate(program.predictors()):+ pred_instructions = get_signature(pred).instructions+ instruction_list.append(f"\"{pred_instructions}\"")+ # Joining the list into a single string that looks like a list+ return f"[{', '.join(instruction_list)}]"++def create_predictor_level_history_string(base_program, predictor_i, trial_logs, top_n):+ instruction_aggregate = {}+ instruction_history = []++ # Load trial programs+ for trial_num in trial_logs:+ trial = trial_logs[trial_num]+ if "program_path" in trial:+ trial_program = base_program.deepcopy()+ trial_program.load(trial["program_path"])+ instruction_history.append({+ "program": trial_program,+ "score": trial["score"]+ })++ # Aggregate scores for each instruction+ for history_item in instruction_history:+ predictor = history_item["program"].predictors()[predictor_i]+ instruction = get_signature(predictor).instructions+ score = history_item["score"]++ if instruction in instruction_aggregate:+ instruction_aggregate[instruction]['total_score'] += score+ instruction_aggregate[instruction]['count'] += 1+ else:+ instruction_aggregate[instruction] = {'total_score': score, 'count': 1}++ # Calculate average score for each instruction and prepare for sorting+ predictor_history = []+ for instruction, data in instruction_aggregate.items():+ average_score = data['total_score'] / data['count']+ predictor_history.append((instruction, average_score))++ # Deduplicate and sort by average score, then select top N+ seen_instructions = set()+ unique_predictor_history = []+ for instruction, score in predictor_history:+ if instruction not in seen_instructions:+ seen_instructions.add(instruction)+ unique_predictor_history.append((instruction, score))++ top_instructions = sorted(unique_predictor_history, key=lambda x: x[1], reverse=True)[:top_n]+ top_instructions.reverse()++ # Create formatted history string+ predictor_history_string = ""+ for instruction, score in top_instructions:+ predictor_history_string += instruction + f" | Score: {score}\n\n"++ return predictor_history_string++def create_example_string(fields, example):++ # Building the output string+ output = []+ for field_name, field_values in fields.items():+ name = field_values.json_schema_extra["prefix"]++ # Determine the value from input_data or prediction_data+ value = example.get(field_name)++ # Construct the string for the current field+ field_str = f"{name} {value}"+ output.append(field_str)++ # Joining all the field strings+ return '\n'.join(output)++# def create_predictor_level_history_string(base_program, predictor_i, trial_logs, top_n):+# instruction_aggregate = {}++# instruction_history = []+# for trial_num in trial_logs:+# trial = trial_logs[trial_num]+# if "program_path" in trial:+# trial_program = base_program.deepcopy()+# trial_program.load(trial["program_path"])+# instruction_history.append({+# "program": trial_program,+# "score": trial["score"]+# })++# for history_item in instruction_history:+# predictor = history_item["program"].predictors()[predictor_i]+# instruction = get_signature(predictor).instructions+# score = history_item["score"]++# if instruction in instruction_aggregate:+# instruction_aggregate[instruction]['total_score'] += score+# instruction_aggregate[instruction]['count'] += 1+# else:+# instruction_aggregate[instruction] = {'total_score': score, 'count': 1}++# # # Calculate average score for each instruction and prepare for sorting+# # predictor_history = []+# # for instruction, data in instruction_aggregate.items():+# # average_score = data['total_score'] / data['count']+# # predictor_history.append((instruction, average_score))+# # Calculate average score for each instruction and prepare for sorting+# predictor_history = []+# seen_instructions = set()+# for instruction, data in instruction_aggregate.items():+# if instruction not in seen_instructions:+# seen_instructions.add(instruction)+# average_score = data['total_score'] / data['count']+# predictor_history.append((instruction, average_score))++# # Sort by average score and select top N+# top_instructions = sorted(predictor_history, key=lambda x: x[1], reverse=True)[:top_n]+# top_instructions.reverse()++# # Create formatted history string+# predictor_history_string = ""+# for instruction, score in top_instructions:+# predictor_history_string += instruction + f" | Score: {score}\n\n"++# return predictor_history_string\ No newline at end of filediff --git a/dspy/teleprompt/hard_bootstrap.py b/dspy/teleprompt/hard_bootstrap.pynew file mode 100644index 000000000..73f050730--- /dev/null+++ b/dspy/teleprompt/hard_bootstrap.py@@ -0,0 +1,240 @@+import dsp+import tqdm+import random+import threading+from dspy.predict.retry import Retry++from dspy.primitives import Example++from .teleprompt import Teleprompter+from .vanilla import LabeledFewShot++from dspy.evaluate.evaluate import Evaluate++# TODO: metrics should return an object with __bool__ basically, but fine if they're more complex.+# They can also be sortable.++# TODO: Switch here from dsp.Example to dspy.Example. Right now, it's okay because it's internal only (predictors).+# NOTE: Notice the places where we don't shuffle examples. I do like that this one doesn't shuffle.+# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when considering candidates.++# TODO: the max_rounds via branch_idx to get past the cache, not just temperature.+# In principle, we can also sample multiple outputs from the final generation step+# (or even each step, in case the validation function just wants *one* thing that works, but nah)+# and try them all. Having a pretty solid guess on the "final step" of each example isn't hard by the second round,+# in the sense that we have the trace from the first round. (Yes it may change but that's an edge case that+# won't hurt our "best effort" guarantees.)++# TODO: When this bootstraps for another teleprompter like finetune, we want all demos we gather.+# But when it's for direct use we may want to sample ONE demo per predictor--example pair. This is important for "multi-use" modules.++# TODO: Add baselines=[...]+++class BootstrapHardFewShot(Teleprompter):+ def __init__(self, metric=None, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, max_errors=5, temperature=0.7, min_failures=2):+ self.metric = metric+ self.teacher_settings = teacher_settings++ self.max_bootstrapped_demos = max_bootstrapped_demos+ self.max_labeled_demos = max_labeled_demos+ self.temperature = temperature+ self.max_rounds = max_rounds+ self.max_errors= max_errors+ self.error_count = 0+ self.min_failures = min_failures+ self.error_lock = threading.Lock()++ def compile(self, student, *, teacher=None, trainset, valset=None):+ self.trainset = trainset+ self.valset = valset++ raise Exception("NOTE THAT THIS HAS BEEN EXPENSIVE TO RUN. CHECK IN WITH KRISTAOO BEFORE DOING A FULL RUN.")++ self._prepare_student_and_teacher(student, teacher)+ self._prepare_predictor_mappings()+ # self._bootstrap()+ #breakpoint()+ self._bootstrap_hard_examples()++ self.student = self._train()+ self.student._compiled = True++ return self.student++ def _prepare_student_and_teacher(self, student, teacher):+ self.student = student.reset_copy()+ self.teacher = teacher.deepcopy() if teacher is not None else student.reset_copy()++ assert self.student._compiled is False, "Student must be uncompiled."++ if self.max_labeled_demos and self.teacher._compiled is False:+ teleprompter = LabeledFewShot(k=self.max_labeled_demos)+ self.teacher = teleprompter.compile(self.teacher.reset_copy(), trainset=self.trainset)++ def _prepare_predictor_mappings(self):+ name2predictor, predictor2name = {}, {}+ student, teacher = self.student, self.teacher++ assert len(student.predictors()) == len(teacher.predictors()), "Student and teacher must have the same number of predictors."++ for (name1, predictor1), (name2, predictor2) in zip(student.named_predictors(), teacher.named_predictors()):+ assert name1 == name2, "Student and teacher must have the same program structure."+ assert predictor1.signature == predictor2.signature, f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"+ assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."++ name2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)+ predictor2name[id(predictor1)] = name1++ # FIXME(shangyint): This is an ugly hack to bind traces of+ # retry.module to retry+ if isinstance(predictor1, Retry):+ predictor2name[id(predictor1.module)] = name1++ predictor2name[id(predictor2)] = name2++ self.name2predictor = name2predictor+ self.predictor2name = predictor2name++ def _bootstrap_hard_examples(self, *, max_bootsraps=None):+ max_bootsraps = max_bootsraps or self.max_bootstrapped_demos++ bootstrapped = {}+ self.name2traces = {name: [] for name in self.name2predictor}++ hard_examples = []+ # start by getting our pool of hard examples. we'll define these as examples that fail >= 3/5 times+ total_attempts = 5+ min_failed_attempts = self.min_failures++ for round_idx in range(self.max_rounds):+ for example_idx, example in enumerate(tqdm.tqdm(self.trainset)):+ total_successes = 0+ successful_attempt = None+ add_example_to_traces = True++ # Try # attempt times to see what % of attempts we get correct+ for i in range(total_attempts):+ if total_successes > 0:+ add_example_to_traces = False # We only need one successful demo for this example+ print(f"attempting bootstrap for {i+1}th time.")+ success, predictor_name, demo = self._bootstrap_one_example(example, i)+ print(f"success of bootstrap: {success}")+ if success:+ successful_attempt = success+ total_successes += int(success)+ if total_successes > (total_attempts - min_failed_attempts):+ break+ # if this example failed the right number of times and is therefore 'hard', let's add it as one of our demo examples+ # print(f"Total Successes: {total_successes} | Total Failures: {total_attempts-total_successes}")+ if (total_attempts-total_successes) / total_attempts > min_failed_attempts/total_attempts and total_successes > 0:+ if predictor_name not in self.name2traces:+ self.name2traces[predictor_name] = []+ self.name2traces[predictor_name].append(demo)+ bootstrapped[example_idx] = True+ print(f"Found {len(bootstrapped)}/{max_bootsraps} hard example(s).")++ if len(bootstrapped) >= max_bootsraps:+ print(f"Bootstrapping complete!")+ break++ # hard_examples.append(example)+ #breakpoint()+ print(f'Bootstrapped {len(bootstrapped)} full traces after {example_idx+1} examples in round {round_idx}.')++ # Unbootstrapped training examples++ self.validation = [x for idx, x in enumerate(self.trainset) if idx not in bootstrapped]+ random.Random(0).shuffle(self.validation)++ self.validation = self.valset or self.validation++ def _bootstrap_one_example(self, example, round_idx=0):+ name2traces = self.name2traces+ teacher = self.teacher #.deepcopy()+ predictor_cache = {}+ predictor_name, demo = None, None++ try:+ with dsp.settings.context(trace=[], **self.teacher_settings):+ # breakpoint()+ lm = dsp.settings.lm+ # breakpoint()+ # lm = lm.copy(temperature=self.temperature + 0.001 * round_idx) if round_idx > 0 else lm+ original_lm_temp = lm.kwargs["temperature"]+ if round_idx > 0:+ lm.kwargs["temperature"] = self.temperature + 0.1 * round_idx+ new_settings = dict(lm=lm) if round_idx > 0 else {}+ # print(f'bootstrapping with temperature {lm.kwargs["temperature"]}')++ with dsp.settings.context(**new_settings):+ for name, predictor in teacher.named_predictors():+ predictor_cache[name] = predictor.demos+ predictor.demos = [x for x in predictor.demos if x != example]++ prediction = teacher(**example.inputs())+ # print(f"Example: {example}")+ # print(f"Prediction: {prediction}")+ trace = dsp.settings.trace++ for name, predictor in teacher.named_predictors():+ predictor.demos = predictor_cache[name]++ success = (self.metric is None) or self.metric(example, prediction, trace)++ lm.kwargs["temperature"] = original_lm_temp+ # print(success, example, prediction)+ except Exception as e:+ success = False+ with self.error_lock:+ self.error_count += 1+ current_error_count = self.error_count+ if current_error_count >= self.max_errors:+ raise e+ print(f'Failed to run or to evaluate example {example} with {self.metric} due to {e}.')++ if success:+ for step in trace:+ predictor, inputs, outputs = step++ if 'dspy_uuid' in example:+ demo = Example(augmented=True, dspy_uuid=example.dspy_uuid, **inputs, **outputs)+ else:+ # TODO: FIXME: This is a hack. RandomSearch will complain for now in this edge case.+ demo = Example(augmented=True, **inputs, **outputs)++ try:+ predictor_name = self.predictor2name[id(predictor)]+ except KeyError as e:+ continue # FIXME: !++ # TODO: Look closer into this. It's a bit tricky to reproduce.+ print(f'Failed to find predictor {predictor} in {self.predictor2name}.')+ print('Are you doing this in a notebook (Jupyter)? This might be caused by redefining values by rerunning cells.')+ print('Try restarting the notebook, or open an issue.')+ raise KeyError(f'Failed to find predictor {id(predictor)} {predictor} in {self.predictor2name}.') from e++ # if add_example_to_traces:+ # name2traces[predictor_name].append(demo)++ return success, predictor_name, demo++ def _train(self):+ rng = random.Random(0)+ raw_demos = self.validation++ for name, predictor in self.student.named_predictors():+ augmented_demos = self.name2traces[name][:self.max_bootstrapped_demos]++ sample_size = min(self.max_labeled_demos - len(augmented_demos), len(raw_demos))+ sample_size = max(0, sample_size)++ raw_demos = rng.sample(raw_demos, sample_size)++ import dspy+ if dspy.settings.release >= 20230928:+ predictor.demos = raw_demos + augmented_demos+ else:+ predictor.demos = augmented_demos + raw_demos++ return self.studentdiff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.pyindex 6254951c1..4e2b14ff5 100644--- a/dspy/teleprompt/mipro_optimizer.py+++ b/dspy/teleprompt/mipro_optimizer.py@@ -1,19 +1,21 @@-import math+from dspy.teleprompt.teleprompt import Teleprompter+from dspy.evaluate.evaluate import Evaluate+from dspy.propose import GroundedProposerimport random+import logging+import os+import pickle+import optuna+from dspy.teleprompt.utils import get_dspy_source_code, setup_logging, save_file_to_log_dir, get_signature, set_signature, print_full_program, eval_candidate_program, save_candidate_program, get_program_with_highest_avg_score, print_full_program, create_n_fewshot_demo_sets, get_task_model_history_for_full_example+from collections import defaultdict+import numpy as npimport sysimport textwrap-from collections import defaultdict-from typing import Any--import optuna-import dsp-import dspy-from dspy.evaluate.evaluate import Evaluate-from dspy.signatures import Signature-from dspy.signatures.signature import signature_to_template-from dspy.teleprompt import BootstrapFewShot-from dspy.teleprompt.teleprompt import Teleprompter+try:+ import wandb+except ImportError:+ wandb = None"""USAGE SUGGESTIONS:@@ -25,7 +27,7 @@ from dspy.teleprompt import MIPROteleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_batches=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)```@@ -33,6 +35,7 @@ Note that this teleprompter takes in the following parameters:* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* teacher_settings: The settings used for the teacher model. When unspecified, defaults to the settings set in settings (i.e., dspy.settings.configure(lm=task_model)).* metric: The task metric used for optimization.* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.@@ -44,322 +47,77 @@ Note that this teleprompter takes in the following parameters:* score: the last average evaluated score for the program* pruned: whether or not this program was prunedThis information will be returned as attributes of the best program.+* log_dir: The directory to save logs and other information to. If unspecified, no logs will be saved.+* view_data_batch_size: The number of examples to view in the data batch when producing the dataset summary. Default=10.+* minibatch_size: The size of the minibatch to use when evaluating the program if using minibatched evaluations. Default=25.+* minibatch_full_eval_steps: The number of steps to take before doing a full evaluation of the program if using minibatched evaluations. Default=10."""+BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3+LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0-class BasicGenerateInstruction(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithDataObservations(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- observations = dspy.InputField(desc="Observations about the dataset and task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithExamples(dspy.Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.-- Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.-- Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- observations = dspy.InputField(desc="Observations about the dataset and task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(- desc="The string at the end of the prompt, which will help the model start solving the task",- )---class ObservationSummarizer(dspy.Signature):- """Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."""-- observations = dspy.InputField(desc="Observations I have made about my dataset")- summary = dspy.OutputField(- desc="Two to Three sentence summary of only the most significant highlights of my observations",- )---class DatasetDescriptor(dspy.Signature):- (- """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""- )-- examples = dspy.InputField(desc="Sample data points from the dataset")- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")---class DatasetDescriptorWithPriorObservations(dspy.Signature):- (- """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""- )-- examples = dspy.InputField(desc="Sample data points from the dataset")- prior_observations = dspy.InputField(desc="Some prior observations I made about the data")- observations = dspy.OutputField(- desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add",- )-+MB_FULL_EVAL_STEPS = 10+MINIBATCH_SIZE = 25#50class MIPRO(Teleprompter):def __init__(self,- metric,prompt_model=None,task_model=None,teacher_settings={},num_candidates=10,- init_temperature=1.0,+ metric=None,+ init_temperature=1.4,verbose=False,track_stats=True,+ log_dir=None,view_data_batch_size=10,+ minibatch_size=MINIBATCH_SIZE,+ minibatch_full_eval_steps=MB_FULL_EVAL_STEPS,):- self.num_candidates = num_candidates+ self.n = num_candidatesself.metric = metricself.init_temperature = init_temperature- self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm- self.task_model = task_model if task_model is not None else dspy.settings.lm+ self.prompt_model = prompt_model+ self.task_model = task_modelself.verbose = verboseself.track_stats = track_stats- self.teacher_settings = teacher_settings+ self.log_dir = log_dirself.view_data_batch_size = view_data_batch_size-- def _print_full_program(self, program):- for i, predictor in enumerate(program.predictors()):- if self.verbose:- print(f"Predictor {i}")- if self.verbose:- print(f"i: {self._get_signature(predictor).instructions}")- *_, last_field = self._get_signature(predictor).fields.values()- if self.verbose:- print(f"p: {last_field.json_schema_extra['prefix']}")- if self.verbose:- print("\n")-- def _print_model_history(self, model, n=1):- if self.verbose:- print(f"Model ({model}) History:")- model.inspect_history(n=n)-- def _observe_data(self, trainset, max_iterations=10):- upper_lim = min(len(trainset), self.view_data_batch_size)- observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))- observations = observation["observations"]-- skips = 0- iterations = 0- for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):- upper_lim = min(len(trainset), b + self.view_data_batch_size)- output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(- prior_observations=observations,- examples=(trainset[b:upper_lim].__repr__()),- )- iterations += 1- if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":- skips += 1- if skips >= 5:- break- continue- if iterations >= max_iterations:- break- observations += output["observations"]-- summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)-- return summary.summary-- def _create_example_string(self, fields, example):- # Building the output string- output = []- for field in fields:- name = field.name- separator = field.separator- input_variable = field.input_variable-- # Determine the value from input_data or prediction_data- value = example.get(input_variable)-- # Construct the string for the current field- field_str = f"{name}{separator}{value}"- output.append(field_str)-- # Joining all the field strings- return "\n".join(output)-- def _get_signature(self, predictor):- if hasattr(predictor, "extended_signature"):- return predictor.extended_signature- elif hasattr(predictor, "signature"):- return predictor.signature- return None-- def _set_signature(self, predictor, updated_signature):- if hasattr(predictor, "extended_signature"):- predictor.extended_signature = updated_signature- elif hasattr(predictor, "signature"):- predictor.signature = updated_signature-- def _generate_first_N_candidates( # noqa: N802+ self.teacher_settings = teacher_settings+ self.prompt_model_total_calls = 0+ self.total_calls = 0+ self.minibatch_size = minibatch_size+ self.minibatch_full_eval_steps = minibatch_full_eval_steps++ # Check if WANDB_RUN_ID is set in the environment+ self.wandb_run_id = None++ def _get_batch_size(self,- module: dspy.Module,- N: int, # noqa: N803- view_data: bool,- view_examples: bool,- demo_candidates: dict,- devset,- ) -> tuple[dict, dict]:- candidates = {}- evaluated_candidates = defaultdict(dict)-- if view_data:- # Create data observations- self.observations = None- with dspy.settings.context(lm=self.prompt_model):- self.observations = self._observe_data(devset).replace("Observations:", "").replace("Summary:", "")-- if view_examples:- example_sets = {}- for predictor in module.predictors():- # Get all augmented examples- example_set = {}- all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples- for example_set_i, set_of_examples in enumerate(all_sets_of_examples):- if example_set_i != 0: # Skip the no examples case- for example in set_of_examples: # Get each individual example in the set- if "augmented" in example and example["augmented"]:- if example_set_i not in example_set:- example_set[example_set_i] = []- fields_to_use = signature_to_template(predictor.signature).fields- _input_variable_names = list(self._get_signature(predictor).input_fields.keys())- example_string = self._create_example_string(fields_to_use, example)- example_set[example_set_i].append(example_string)- example_sets[id(predictor)] = example_set- else:- example_set[example_set_i] = []- example_sets[id(predictor)] = example_set-- # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts- for predictor in module.predictors():- basic_instruction = None- basic_prefix = None- basic_instruction = self._get_signature(predictor).instructions- *_, last_field = self._get_signature(predictor).fields.values()- basic_prefix = last_field.json_schema_extra["prefix"]- with dspy.settings.context(lm=self.prompt_model):- # Data & Examples- if view_data and view_examples:- if 1 not in example_sets[id(predictor)]:- raise ValueError("No examples found for the given predictor")- instruct = None- for i in range(1, self.num_candidates):- new_instruct = dspy.Predict(- BasicGenerateInstructionWithExamplesAndDataObservations,- n=1,- temperature=self.init_temperature,- )(- basic_instruction=basic_instruction,- observations=self.observations,- examples=example_sets[id(predictor)][i],- )- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(- new_instruct.completions.proposed_instruction,- )- instruct.completions.proposed_prefix_for_output_field.extend(- new_instruct.completions.proposed_prefix_for_output_field,- )- # Just data- elif view_data:- instruct = dspy.Predict(- BasicGenerateInstructionWithDataObservations,- n=N - 1,- temperature=self.init_temperature,- )(basic_instruction=basic_instruction, observations=self.observations)- # Just examples- elif view_examples:- instruct = None- for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty- new_instruct = dspy.Predict(- BasicGenerateInstructionWithExamples,- n=1,- temperature=self.init_temperature,- )(- basic_instruction=basic_instruction,- examples=example_sets[id(predictor)][i],- )- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(- new_instruct.completions.proposed_instruction,- )- instruct.completions.proposed_prefix_for_output_field.extend(- new_instruct.completions.proposed_prefix_for_output_field,- )- # Neither- else:- instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)(- basic_instruction=basic_instruction,- )-- # Add in our initial prompt as a candidate as well- instruct.completions.proposed_instruction.insert(0, basic_instruction)- instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)- candidates[id(predictor)] = instruct.completions- evaluated_candidates[id(predictor)] = {}-- if self.verbose:- self._print_model_history(self.prompt_model)-- return candidates, evaluated_candidates+ minibatch,+ trainset,+ ):+ if minibatch:+ return self.minibatch_size+ else:+ return len(trainset)def compile(self,- student: dspy.Program,+ student,*,- trainset: list[dspy.Example],- num_trials: int,- max_bootstrapped_demos: int,- max_labeled_demos: int,- eval_kwargs: dict[str, Any],- seed=42,- view_data=True,- view_examples=True,+ trainset,+ valset=[],+ num_batches=30,+ max_bootstrapped_demos=5,+ max_labeled_demos=2,+ eval_kwargs={},+ seed=9,+ minibatch=True,+ program_aware_proposer=True,requires_permission_to_run=True,- ) -> dspy.Program:+ ):# Define ANSI escape codes for colorsYELLOW = "\033[93m"BLUE = "\033[94m"@@ -368,18 +126,34 @@ class MIPRO(Teleprompter):random.seed(seed)- estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P- estimated_prompt_model_calls = 10 + self.num_candidates * len(- student.predictors(),- ) # num data summary calls + N * P+ estimated_prompt_model_calls = 10 + self.n * len(+ student.predictors(),+ ) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)++ prompt_model_line = ""+ if not program_aware_proposer:+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""+ else:+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""++ estimated_task_model_calls_wo_module_calls = 0+ task_model_line = ""+ if not minibatch:+ estimated_task_model_calls_wo_module_calls = len(trainset) * num_batches # M * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""+ else:+ estimated_task_model_calls_wo_module_calls = self.minibatch_size * num_batches + (len(trainset) * (num_batches // self.minibatch_full_eval_steps)) # B * T * P+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{self.minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches // self.minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""+user_message = textwrap.dedent(f"""\{YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:- {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}- {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}++ {prompt_model_line}+ {task_model_line}{YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}@@ -389,7 +163,7 @@ class MIPRO(Teleprompter):For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the taskand prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:- {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}+ {YELLOW}- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.{ENDC}{YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")user_confirmation_message = textwrap.dedent(f"""\@@ -413,186 +187,308 @@ class MIPRO(Teleprompter):run = Falseif run:+ if program_aware_proposer:+ try:+ self.program_code_string = get_dspy_source_code(student)+ if self.verbose:+ print("SOURCE CODE:",self.program_code_string)+ except Exception as e:+ print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")+ self.program_code_string = None+ program_aware_proposer = False++ # Setup our proposer+ proposer = GroundedProposer(+ trainset=trainset,+ prompt_model=self.prompt_model,+ program_code_string=self.program_code_string,+ program_aware=program_aware_proposer,+ )++ # Setup logging+ logging.basicConfig(level=logging.WARNING)+ setup_logging(self.log_dir)++ self.wandb_run_id = os.getenv("WANDB_RUN_ID", None)++ if self.wandb_run_id:+ # Initialize wandb with the same run ID+ wandb.init(+ project="prompt_optimizers",+ id=self.wandb_run_id,+ resume="must",+ )++ # Setup random seeds+ random.seed(seed)+ np.random.seed(seed)++ # Log current file to log_dir+ curr_file = os.path.abspath(__file__)+ save_file_to_log_dir(curr_file, self.log_dir)+# Set up program and evaluation function- module = student.deepcopy()+ program = student.deepcopy()evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)- # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt- if (- max_bootstrapped_demos == 0 and max_labeled_demos == 0- ): # TODO: address case when max_bootstrapped alone is 0- max_bootstrapped_demos_for_candidate_gen = 1- max_labeled_demos_for_candidate_gen = 1 # TODO: this might only need to be 0+ # Determine the number of fewshot examples to use to generate demos for prompt+ if max_bootstrapped_demos == 0 and max_labeled_demos == 0:+ max_bootstrapped_demos_for_candidate_gen = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT+ max_labeled_demos_for_candidate_gen = LABELED_FEWSHOT_EXAMPLES_IN_CONTEXTelse:max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demosmax_labeled_demos_for_candidate_gen = max_labeled_demos- # Generate N few shot example sets- demo_candidates = {}- for i in range(self.num_candidates):- if i == 0: # Story empty set of demos as default for index 0- for module_p in module.predictors():- if id(module_p) not in demo_candidates:- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append([])- else:- if self.verbose:- print(f"Creating basic bootstrap: {i}/{self.num_candidates-1}")-- # Create a new basic bootstrap few - shot program .- rng = random.Random(i)- shuffled_trainset = trainset[:] # Create a copy of devset- rng.shuffle(shuffled_trainset) # Shuffle the copy- tp = BootstrapFewShot(- metric=self.metric,- max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,- max_labeled_demos=max_labeled_demos_for_candidate_gen,- teacher_settings=self.teacher_settings,- )- candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_trainset)+ # # Generate N few shot example sets+ # if not demo_candidates and not instruction_candidates:+ # demo_candidates = create_n_fewshot_example_sets(program=program, trainset=trainset, n=self.n, hard_fewshot=hard_fewshot, metric=self.metric, teacher_settings=self.teacher_settings, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen)- # Store the candidate demos- for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):- if id(module_p) not in demo_candidates:- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append(candidate_p.demos)+ # Generate N few shot example sets+ try:+ demo_candidates = create_n_fewshot_demo_sets(+ student=program,+ num_candidate_sets=self.n,+ trainset=trainset,+ max_labeled_demos=max_labeled_demos_for_candidate_gen,+ max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,+ metric=self.metric,+ teacher_settings=self.teacher_settings,+ seed=seed,+ )+ except Exception as e:+ print(f"Error generating fewshot examples: {e}")+ print("Running without fewshot examples.")+ demo_candidates = None# Generate N candidate prompts- instruction_candidates, _ = self._generate_first_N_candidates(- module,- self.num_candidates,- view_data,- view_examples,- demo_candidates,- trainset,+ proposer.program_aware = program_aware_proposer+ proposer.use_tip = True+ proposer.use_instruct_history = False+ proposer.set_history_randomly = False+ instruction_candidates = proposer.propose_instructions_for_program(+ trainset=trainset,+ program=program,+ demo_candidates=demo_candidates,+ N=self.n,+ prompt_model=self.prompt_model,+ T=self.init_temperature,+ trial_logs={},)+ for i, pred in enumerate(program.predictors()):+ instruction_candidates[i][0] = get_signature(pred).instructions++ # instruction_candidates[1][0] = "Given the question, and context, respond with the number of the document that is most relevant to answering the question in the field 'Answer' (ex. Answer: '3')."++ # Save the candidate instructions generated+ if self.log_dir:+ fp = os.path.join(self.log_dir, "instructions_to_save.pickle")+ with open(fp, "wb") as file:+ pickle.dump(instruction_candidates, file)- # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples+ # If we're doing zero-shot, reset demo_candidates to noneif max_bootstrapped_demos == 0 and max_labeled_demos == 0:demo_candidates = None# Initialize variables to store the best program and its scorebest_score = float("-inf")best_program = None- trial_num = 0-trial_logs = {}+ total_eval_calls = 0+ param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scores+ fully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of# Define our trial objective- def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, trainset):+ def create_objective(+ baseline_program,+ instruction_candidates,+ demo_candidates,+ evaluate,+ trainset,+ ):def objective(trial):- nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables+ nonlocal best_program, best_score, trial_logs, total_eval_calls # Allow access to the outer variables++ # Kick off trial+ logging.info(f"Starting trial num: {trial.number}")+ trial_logs[trial.number] = {}++ # Create a new candidate programcandidate_program = baseline_program.deepcopy()- # Suggest the instruction to use for our predictor- print(f"Starting trial #{trial_num}")- trial_logs[trial_num] = {}+ # Choose set of instructions & demos to use for each predictor+ chosen_params = []+ for i, p_new in enumerate(candidate_program.predictors()):- for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):- # Get instruction candidates for our given predictor- p_instruction_candidates = instruction_candidates[id(p_old)]+ # Get instruction candidates / demos for our given predictor+ p_instruction_candidates = instruction_candidates[i]if demo_candidates:- p_demo_candidates = demo_candidates[id(p_old)]+ p_demo_candidates = demo_candidates[i]- # Suggest the index of the instruction candidate to use in our trial+ # Suggest the index of the instruction / demo candidate to use in our trialinstruction_idx = trial.suggest_categorical(- f"{id(p_old)}_predictor_instruction",+ f"{i}_predictor_instruction",range(len(p_instruction_candidates)),)+ chosen_params.append(instruction_idx)if demo_candidates:demos_idx = trial.suggest_categorical(- f"{id(p_old)}_predictor_demos",- range(len(p_demo_candidates)),+ f"{i}_predictor_demos", range(len(p_demo_candidates)))- trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx+ chosen_params.append(demo_candidates)++ # Log the selected instruction / demo candidate+ trial_logs[trial.number][+ f"{i}_predictor_instruction"+ ] = instruction_idxif demo_candidates:- trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx-- # Get the selected instruction candidate- selected_candidate = p_instruction_candidates[instruction_idx]- selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()- selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()-- # Use this candidates in our program- *_, last_field = self._get_signature(p_new).fields.keys()- updated_signature = (- self._get_signature(p_new)- .with_instructions(selected_instruction)- .with_updated_fields(last_field, prefix=selected_prefix)- )- self._set_signature(p_new, updated_signature)+ trial_logs[trial.number][f"{i}_predictor_demos"] = demos_idx- # Get the selected demos+ logging.info(f"instruction_idx {instruction_idx}")if demo_candidates:- selected_demos = p_demo_candidates[demos_idx]+ logging.info(f"demos_idx {demos_idx}")- # Use these demos in our program+ # Set the instruction+ selected_instruction = p_instruction_candidates[instruction_idx]+ updated_signature = get_signature(p_new).with_instructions(+ selected_instruction+ )+ set_signature(p_new, updated_signature)++ # Set the demosif demo_candidates:- p_new.demos = selected_demos+ p_new.demos = p_demo_candidates[demos_idx]- if self.verbose:- print("Evaling the following program:")- if self.verbose:- self._print_full_program(candidate_program)- trial_logs[trial_num]["program"] = candidate_program-- # Evaluate with the new prompts- total_score = 0- batch_size = 100- num_batches = math.ceil(len(trainset) / batch_size)-- for i in range(num_batches):- start_index = i * batch_size- end_index = min((i + 1) * batch_size, len(trainset))- split_trainset = trainset[start_index:end_index]- split_score = evaluate(candidate_program, devset=split_trainset, display_table=0)- if self.verbose:- print(f"{i}st split score: {split_score}")-- total_score += split_score * len(split_trainset)- curr_weighted_avg_score = total_score / min((i + 1) * 100, len(trainset))- if self.verbose:- print(f"curr average score: {curr_weighted_avg_score}")-- trial.report(curr_weighted_avg_score, i)-- # Handle pruning based on the intermediate value.- if trial.should_prune():- print("Trial pruned.")- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = True- trial_num += 1- raise optuna.TrialPruned()+ # Log assembled program+ print(f"CANDIDATE PROGRAM:")+ print_full_program(candidate_program)+ print(f"...")- if self.verbose:- print(f"Fully evaled score: {curr_weighted_avg_score}")- if self.verbose:- self._print_model_history(self.task_model, n=1)- score = curr_weighted_avg_score+ # Save the candidate program+ trial_logs[trial.number]["program_path"] = save_candidate_program(+ candidate_program, self.log_dir, trial.number+ )++ trial_logs[trial.number]["num_eval_calls"] = 0++ # Evaluate the candidate program with relevant batch size+ batch_size = self._get_batch_size(minibatch, trainset)+ score = eval_candidate_program(+ batch_size, trainset, candidate_program, evaluate+ )- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = False+ # Print out a full trace of the program in use+ print(f"FULL TRACE")+ full_trace = get_task_model_history_for_full_example(+ candidate_program, self.task_model, trainset, evaluate+ )+ print(f"...")- # Update the best program if the current score is better- if score > best_score:+ # Log relevant information+ print(f"Score {score}")+ categorical_key = ",".join(map(str, chosen_params))+ param_score_dict[categorical_key].append(+ (score, candidate_program)+ )+ trial_logs[trial.number]["num_eval_calls"] = batch_size+ trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)+ trial_logs[trial.number]["eval_example_call"] = full_trace+ trial_logs[trial.number]["score"] = score+ trial_logs[trial.number]["pruned"] = False+ total_eval_calls += trial_logs[trial.number]["num_eval_calls"]+ trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number]["program"] = candidate_program.deepcopy()+ if self.wandb_run_id:+ wandb.log(+ {+ "score": score,+ "num_eval_calls": trial_logs[trial.number]["num_eval_calls"],+ "total_eval_calls": total_eval_calls,+ }+ )++ # Update the best program if the current score is better, and if we're not using minibatching+ best_score_updated = False+ if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:+ print("Updating best score")best_score = scorebest_program = candidate_program.deepcopy()+ best_score_updated = True+++ # If we're doing minibatching, check to see if it's time to do a full eval+ if minibatch and trial.number % self.minibatch_full_eval_steps == 0:++ # Save old information as the minibatch version+ trial_logs[trial.number]["mb_score"] = score+ trial_logs[trial.number]["mb_program_path"] = trial_logs[trial.number]["program_path"]++ # Identify our best program (based on mean of scores so far, and do a full eval on it)+ highest_mean_program, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)+ full_train_score = eval_candidate_program(+ len(trainset), trainset, highest_mean_program, evaluate+ )- trial_num += 1+ # Log relevant information+ fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}+ total_eval_calls += len(trainset)+ trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial.number]["full_eval"] = True+ trial_logs[trial.number]["program_path"] = save_candidate_program(+ program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number, note="full_eval"+ )+ trial_logs[trial.number]["score"] = full_train_score++ if full_train_score > best_score:+ print(f"UPDATING BEST SCORE WITH {full_train_score}")+ best_score = full_train_score+ best_program = highest_mean_program.deepcopy()+ best_score_updated = True++ # If the best score was updated, do a full eval on the dev set+ if best_score_updated:+ full_dev_score = evaluate(+ best_program,+ devset=valset,+ display_table=0,+ )+ if self.wandb_run_id:+ wandb.log(+ {+ "best_prog_so_far_train_score": best_score,+ "best_prog_so_far_dev_score": full_dev_score,+ }+ )return scorereturn objective# Run the trial- objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)- sampler = optuna.samplers.TPESampler(seed=seed)+ objective_function = create_objective(+ program, instruction_candidates, demo_candidates, evaluate, trainset+ )++ # if minibatch:+ # num_batches = int(num_batches*len(trainset)/(self.minibatch_size + len(trainset)/self.minibatch_full_eval_steps))++ sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)study = optuna.create_study(direction="maximize", sampler=sampler)- _score = study.optimize(objective_function, n_trials=num_trials)+ score = study.optimize(objective_function, n_trials=num_batches)if best_program is not None and self.track_stats:best_program.trial_logs = trial_logs+ best_program.score = best_score+ best_program.prompt_model_total_calls = self.prompt_model_total_calls+ best_program.total_calls = self.total_calls++ # program_file_path = os.path.join(self.log_dir, 'best_program.pickle')+ if self.log_dir:+ program_file_path = os.path.join(self.log_dir, "best_program")+ best_program.save(program_file_path)++ optuna_study_file_path = os.path.join(self.log_dir, "optuna_study.pickle")+ with open(optuna_study_file_path, "wb") as file:+ pickle.dump(study, file)- print(f"Returning {best_program} from continue_program")return best_program- return None+ return studentdiff --git a/dspy/teleprompt/mipro_optimizer_deprecated.py b/dspy/teleprompt/mipro_optimizer_deprecated.pynew file mode 100644index 000000000..034ea1ef3--- /dev/null+++ b/dspy/teleprompt/mipro_optimizer_deprecated.py@@ -0,0 +1,598 @@+import math+import random+import sys+import textwrap+from collections import defaultdict+from typing import Any++import optuna++import dsp+import dspy+from dspy.evaluate.evaluate import Evaluate+from dspy.signatures import Signature+from dspy.signatures.signature import signature_to_template+from dspy.teleprompt import BootstrapFewShot+from dspy.teleprompt.teleprompt import Teleprompter++"""+USAGE SUGGESTIONS:++The following code can be used to compile a optimized signature teleprompter using MIPRO, and evaluate it on an end task:++``` python+from dspy.teleprompt import MIPRO++teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)+kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)+eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)+```++Note that this teleprompter takes in the following parameters:++* prompt_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* task_model: The model used for prompt generation. When unspecified, defaults to the model set in settings (i.e., dspy.settings.configure(lm=task_model)).+* metric: The task metric used for optimization.+* num_candidates: The number of new prompts and sets of fewshot examples to generate and evaluate. Default=10.+* init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0.+* verbose: Tells the method whether or not to print intermediate steps.+* track_stats: Tells the method whether or not to track statistics about the optimization process.+ If True, the method will track a dictionary with a key corresponding to the trial number,+ and a value containing a dict with the following keys:+ * program: the program being evaluated at a given trial+ * score: the last average evaluated score for the program+ * pruned: whether or not this program was pruned+ This information will be returned as attributes of the best program.+"""+++class BasicGenerateInstruction(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithDataObservations(Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ observations = dspy.InputField(desc="Observations about the dataset and task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithExamples(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.++ Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):+ """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.++ Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative."""++ observations = dspy.InputField(desc="Observations about the dataset and task")+ examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")+ proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")+ proposed_prefix_for_output_field = dspy.OutputField(+ desc="The string at the end of the prompt, which will help the model start solving the task",+ )+++class ObservationSummarizer(dspy.Signature):+ """Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."""++ observations = dspy.InputField(desc="Observations I have made about my dataset")+ summary = dspy.OutputField(+ desc="Two to Three sentence summary of only the most significant highlights of my observations",+ )+++class DatasetDescriptor(dspy.Signature):+ (+ """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""+ )++ examples = dspy.InputField(desc="Sample data points from the dataset")+ observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")+++class DatasetDescriptorWithPriorObservations(dspy.Signature):+ (+ """Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """+ """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """+ """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """+ """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"""+ )++ examples = dspy.InputField(desc="Sample data points from the dataset")+ prior_observations = dspy.InputField(desc="Some prior observations I made about the data")+ observations = dspy.OutputField(+ desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add",+ )+++class MIPRO_DEPRECATED(Teleprompter):+ def __init__(+ self,+ metric,+ prompt_model=None,+ task_model=None,+ teacher_settings={},+ num_candidates=10,+ init_temperature=1.0,+ verbose=False,+ track_stats=True,+ view_data_batch_size=10,+ ):+ self.num_candidates = num_candidates+ self.metric = metric+ self.init_temperature = init_temperature+ self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm+ self.task_model = task_model if task_model is not None else dspy.settings.lm+ self.verbose = verbose+ self.track_stats = track_stats+ self.teacher_settings = teacher_settings+ self.view_data_batch_size = view_data_batch_size++ def _print_full_program(self, program):+ for i, predictor in enumerate(program.predictors()):+ if self.verbose:+ print(f"Predictor {i}")+ if self.verbose:+ print(f"i: {self._get_signature(predictor).instructions}")+ *_, last_field = self._get_signature(predictor).fields.values()+ if self.verbose:+ print(f"p: {last_field.json_schema_extra['prefix']}")+ if self.verbose:+ print("\n")++ def _print_model_history(self, model, n=1):+ if self.verbose:+ print(f"Model ({model}) History:")+ model.inspect_history(n=n)++ def _observe_data(self, trainset, max_iterations=10):+ upper_lim = min(len(trainset), self.view_data_batch_size)+ observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))+ observations = observation["observations"]++ skips = 0+ iterations = 0+ for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):+ upper_lim = min(len(trainset), b + self.view_data_batch_size)+ output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(+ prior_observations=observations,+ examples=(trainset[b:upper_lim].__repr__()),+ )+ iterations += 1+ if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":+ skips += 1+ if skips >= 5:+ break+ continue+ if iterations >= max_iterations:+ break+ observations += output["observations"]++ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)++ return summary.summary++ def _create_example_string(self, fields, example):+ # Building the output string+ output = []+ for field in fields:+ name = field.name+ separator = field.separator+ input_variable = field.input_variable++ # Determine the value from input_data or prediction_data+ value = example.get(input_variable)++ # Construct the string for the current field+ field_str = f"{name}{separator}{value}"+ output.append(field_str)++ # Joining all the field strings+ return "\n".join(output)++ def _get_signature(self, predictor):+ if hasattr(predictor, "extended_signature"):+ return predictor.extended_signature+ elif hasattr(predictor, "signature"):+ return predictor.signature+ return None++ def _set_signature(self, predictor, updated_signature):+ if hasattr(predictor, "extended_signature"):+ predictor.extended_signature = updated_signature+ elif hasattr(predictor, "signature"):+ predictor.signature = updated_signature++ def _generate_first_N_candidates( # noqa: N802+ self,+ module: dspy.Module,+ N: int, # noqa: N803+ view_data: bool,+ view_examples: bool,+ demo_candidates: dict,+ devset,+ ) -> tuple[dict, dict]:+ candidates = {}+ evaluated_candidates = defaultdict(dict)++ if view_data:+ # Create data observations+ self.observations = None+ with dspy.settings.context(lm=self.prompt_model):+ self.observations = self._observe_data(devset).replace("Observations:", "").replace("Summary:", "")++ if view_examples:+ example_sets = {}+ for predictor in module.predictors():+ # Get all augmented examples+ example_set = {}+ all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples+ for example_set_i, set_of_examples in enumerate(all_sets_of_examples):+ if example_set_i != 0: # Skip the no examples case+ for example in set_of_examples: # Get each individual example in the set+ if "augmented" in example and example["augmented"]:+ if example_set_i not in example_set:+ example_set[example_set_i] = []+ fields_to_use = signature_to_template(predictor.signature).fields+ _input_variable_names = list(self._get_signature(predictor).input_fields.keys())+ example_string = self._create_example_string(fields_to_use, example)+ example_set[example_set_i].append(example_string)+ example_sets[id(predictor)] = example_set+ else:+ example_set[example_set_i] = []+ example_sets[id(predictor)] = example_set++ # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts+ for predictor in module.predictors():+ basic_instruction = None+ basic_prefix = None+ basic_instruction = self._get_signature(predictor).instructions+ *_, last_field = self._get_signature(predictor).fields.values()+ basic_prefix = last_field.json_schema_extra["prefix"]+ with dspy.settings.context(lm=self.prompt_model):+ # Data & Examples+ if view_data and view_examples:+ if 1 not in example_sets[id(predictor)]:+ raise ValueError("No examples found for the given predictor")+ instruct = None+ for i in range(1, self.num_candidates):+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamplesAndDataObservations,+ n=1,+ temperature=self.init_temperature,+ )(+ basic_instruction=basic_instruction,+ observations=self.observations,+ examples=example_sets[id(predictor)][i],+ )+ if not instruct:+ instruct = new_instruct+ else:+ instruct.completions.proposed_instruction.extend(+ new_instruct.completions.proposed_instruction,+ )+ instruct.completions.proposed_prefix_for_output_field.extend(+ new_instruct.completions.proposed_prefix_for_output_field,+ )+ # Just data+ elif view_data:+ instruct = dspy.Predict(+ BasicGenerateInstructionWithDataObservations,+ n=N - 1,+ temperature=self.init_temperature,+ )(basic_instruction=basic_instruction, observations=self.observations)+ # Just examples+ elif view_examples:+ instruct = None+ for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamples,+ n=1,+ temperature=self.init_temperature,+ )(+ basic_instruction=basic_instruction,+ examples=example_sets[id(predictor)][i],+ )+ if not instruct:+ instruct = new_instruct+ else:+ instruct.completions.proposed_instruction.extend(+ new_instruct.completions.proposed_instruction,+ )+ instruct.completions.proposed_prefix_for_output_field.extend(+ new_instruct.completions.proposed_prefix_for_output_field,+ )+ # Neither+ else:+ instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)(+ basic_instruction=basic_instruction,+ )++ # Add in our initial prompt as a candidate as well+ instruct.completions.proposed_instruction.insert(0, basic_instruction)+ instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)+ candidates[id(predictor)] = instruct.completions+ evaluated_candidates[id(predictor)] = {}++ if self.verbose:+ self._print_model_history(self.prompt_model)++ return candidates, evaluated_candidates++ def compile(+ self,+ student: dspy.Program,+ *,+ trainset: list[dspy.Example],+ num_trials: int,+ max_bootstrapped_demos: int,+ max_labeled_demos: int,+ eval_kwargs: dict[str, Any],+ seed=42,+ view_data=True,+ view_examples=True,+ requires_permission_to_run=True,+ ) -> dspy.Program:+ # Define ANSI escape codes for colors+ YELLOW = "\033[93m"+ BLUE = "\033[94m"+ BOLD = "\033[1m"+ ENDC = "\033[0m" # Resets the color to default++ random.seed(seed)++ estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P+ estimated_prompt_model_calls = 10 + self.num_candidates * len(+ student.predictors(),+ ) # num data summary calls + N * P++ user_message = textwrap.dedent(f"""\+ {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}++ Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:++ {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}+ {YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}++ {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}++ {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token)+ + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC}++ For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task+ and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:++ {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")++ user_confirmation_message = textwrap.dedent(f"""\+ To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.++ If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}++ {YELLOW}Awaiting your input...{ENDC}+ """)++ print(user_message)++ sys.stdout.flush() # Flush the output buffer to force the message to print++ run = True+ if requires_permission_to_run:+ print(user_confirmation_message)+ user_input = input("Do you wish to continue? (y/n): ").strip().lower()+ if user_input != "y":+ print("Compilation aborted by the user.")+ run = False++ if run:+ # Set up program and evaluation function+ module = student.deepcopy()+ evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)++ # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt+ if (+ max_bootstrapped_demos == 0 and max_labeled_demos == 0+ ): # TODO: address case when max_bootstrapped alone is 0+ max_bootstrapped_demos_for_candidate_gen = 1+ max_labeled_demos_for_candidate_gen = 1 # TODO: this might only need to be 0+ else:+ max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos+ max_labeled_demos_for_candidate_gen = max_labeled_demos++ # Generate N few shot example sets+ demo_candidates = {}+ for i in range(self.num_candidates):+ if i == 0: # Story empty set of demos as default for index 0+ for module_p in module.predictors():+ if id(module_p) not in demo_candidates:+ demo_candidates[id(module_p)] = []+ demo_candidates[id(module_p)].append([])+ else:+ if self.verbose:+ print(f"Creating basic bootstrap: {i}/{self.num_candidates-1}")++ # Create a new basic bootstrap few - shot program .+ rng = random.Random(i)+ shuffled_trainset = trainset[:] # Create a copy of devset+ rng.shuffle(shuffled_trainset) # Shuffle the copy+ tp = BootstrapFewShot(+ metric=self.metric,+ max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,+ max_labeled_demos=max_labeled_demos_for_candidate_gen,+ teacher_settings=self.teacher_settings,+ )+ candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_trainset)++ # Store the candidate demos+ for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):+ if id(module_p) not in demo_candidates:+ demo_candidates[id(module_p)] = []+ demo_candidates[id(module_p)].append(candidate_p.demos)++ # Generate N candidate prompts+ instruction_candidates, _ = self._generate_first_N_candidates(+ module,+ self.num_candidates,+ view_data,+ view_examples,+ demo_candidates,+ trainset,+ )++ # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples+ if max_bootstrapped_demos == 0 and max_labeled_demos == 0:+ demo_candidates = None++ # Initialize variables to store the best program and its score+ best_score = float("-inf")+ best_program = None+ trial_num = 0++ trial_logs = {}++ # Define our trial objective+ def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, trainset):+ def objective(trial):+ nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables+ candidate_program = baseline_program.deepcopy()++ # Suggest the instruction to use for our predictor+ print(f"Starting trial #{trial_num}")+ trial_logs[trial_num] = {}++ for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):+ # Get instruction candidates for our given predictor+ p_instruction_candidates = instruction_candidates[id(p_old)]+ if demo_candidates:+ p_demo_candidates = demo_candidates[id(p_old)]++ # Suggest the index of the instruction candidate to use in our trial+ instruction_idx = trial.suggest_categorical(+ f"{id(p_old)}_predictor_instruction",+ range(len(p_instruction_candidates)),+ )+ if demo_candidates:+ demos_idx = trial.suggest_categorical(+ f"{id(p_old)}_predictor_demos",+ range(len(p_demo_candidates)),+ )+ trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx+ if demo_candidates:+ trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx++ # Get the selected instruction candidate+ selected_candidate = p_instruction_candidates[instruction_idx]+ selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()+ selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()++ # Use this candidates in our program+ *_, last_field = self._get_signature(p_new).fields.keys()+ updated_signature = (+ self._get_signature(p_new)+ .with_instructions(selected_instruction)+ .with_updated_fields(last_field, prefix=selected_prefix)+ )+ self._set_signature(p_new, updated_signature)++ # Get the selected demos+ if demo_candidates:+ selected_demos = p_demo_candidates[demos_idx]++ # Use these demos in our program+ if demo_candidates:+ p_new.demos = selected_demos++ if self.verbose:+ print("Evaling the following program:")+ if self.verbose:+ self._print_full_program(candidate_program)+ trial_logs[trial_num]["program"] = candidate_program++ # Evaluate with the new prompts+ total_score = 0+ batch_size = 100+ num_batches = math.ceil(len(trainset) / batch_size)++ for i in range(num_batches):+ start_index = i * batch_size+ end_index = min((i + 1) * batch_size, len(trainset))+ split_trainset = trainset[start_index:end_index]+ split_score = evaluate(candidate_program, devset=split_trainset, display_table=0)+ if self.verbose:+ print(f"{i}st split score: {split_score}")++ total_score += split_score * len(split_trainset)+ curr_weighted_avg_score = total_score / min((i + 1) * 100, len(trainset))+ if self.verbose:+ print(f"curr average score: {curr_weighted_avg_score}")++ trial.report(curr_weighted_avg_score, i)++ # Handle pruning based on the intermediate value.+ if trial.should_prune():+ print("Trial pruned.")+ trial_logs[trial_num]["score"] = curr_weighted_avg_score+ trial_logs[trial_num]["pruned"] = True+ trial_num += 1+ raise optuna.TrialPruned()++ if self.verbose:+ print(f"Fully evaled score: {curr_weighted_avg_score}")+ if self.verbose:+ self._print_model_history(self.task_model, n=1)+ score = curr_weighted_avg_score++ trial_logs[trial_num]["score"] = curr_weighted_avg_score+ trial_logs[trial_num]["pruned"] = False++ # Update the best program if the current score is better+ if score > best_score:+ best_score = score+ best_program = candidate_program.deepcopy()++ trial_num += 1++ return score++ return objective++ # Run the trial+ objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)+ sampler = optuna.samplers.TPESampler(seed=seed)+ study = optuna.create_study(direction="maximize", sampler=sampler)+ _score = study.optimize(objective_function, n_trials=num_trials)++ if best_program is not None and self.track_stats:+ best_program.trial_logs = trial_logs++ print(f"Returning {best_program} from continue_program")+ return best_program+ return Nonediff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pynew file mode 100644index 000000000..94da21180--- /dev/null+++ b/dspy/teleprompt/utils.py@@ -0,0 +1,433 @@+import dspy+import random+import numpy as np+import logging+import os+import math+import shutil+import numpy as np+from dspy.teleprompt.bootstrap import BootstrapFewShot, LabeledFewShot+from dspy.teleprompt.hard_bootstrap import BootstrapHardFewShot+from dspy.predict.parameter import Parameter++import inspect+import sys+from IPython.core.magics.code import extract_symbols++"""+This file consists of helper functions for our variety of optimizers.+"""++### OPTIMIZER TRAINING UTILS ###+++def create_minibatch(trainset, batch_size=50):+ """Create a minibatch from the trainset."""++ # Ensure batch_size isn't larger than the size of the dataset+ batch_size = min(batch_size, len(trainset))++ # Randomly sample indices for the mini-batch+ sampled_indices = random.sample(range(len(trainset)), batch_size)++ # Create the mini-batch using the sampled indices+ minibatch = [trainset[i] for i in sampled_indices]++ return minibatch+++def eval_candidate_program(batch_size, trainset, candidate_program, evaluate):+ """Evaluate a candidate program on the trainset, using the specified batch size."""+ # Evaluate on the full trainset+ if batch_size >= len(trainset):+ score = evaluate(candidate_program, devset=trainset, display_table=0)+ # Or evaluate on a minibatch+ else:+ score = evaluate(+ candidate_program,+ devset=create_minibatch(trainset, batch_size),+ display_table=0,+ )++ return score+++def eval_candidate_program_with_pruning(+ trial, trial_logs, trainset, candidate_program, evaluate, trial_num, batch_size=100+):+ """Evaluation of candidate_program with pruning implemented"""++ # Evaluate with the new prompts+ total_score = 0+ num_batches = math.ceil(len(trainset) / batch_size)+ total_eval_size = 0++ for i in range(num_batches):+ start_index = i * batch_size+ end_index = min((i + 1) * batch_size, len(trainset))+ split_trainset = trainset[start_index:end_index]+ split_score = evaluate(+ candidate_program, devset=split_trainset, display_table=0+ )+ print(f"{i}st split score: {split_score}")+ total_eval_size += len(split_trainset)++ total_score += split_score * len(split_trainset)+ curr_weighted_avg_score = total_score / min((i + 1) * batch_size, len(trainset))+ print(f"curr average score: {curr_weighted_avg_score}")++ trial.report(curr_weighted_avg_score, i)++ # Handle pruning based on the intermediate value.+ if trial.should_prune():+ print("Trial pruned.")+ trial_logs[trial_num]["score"] = curr_weighted_avg_score+ trial_logs[trial_num]["num_eval_calls"] = total_eval_size+ trial_logs[trial_num]["pruned"] = True+ return curr_weighted_avg_score, trial_logs, total_eval_size, True++ print(f"Fully evaled score: {curr_weighted_avg_score}")+ score = curr_weighted_avg_score++ trial_logs[trial_num]["full_eval"] = False+ trial_logs[trial_num]["score"] = score+ trial_logs[trial_num]["pruned"] = False+ return score, trial_logs, total_eval_size, False+++def get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos):+ """Used as a helper function for bayesian + minibatching optimizers. Returns the program with the highest average score from the batches evaluated so far."""++ # Calculate the mean for each combination of categorical parameters, based on past trials+ results = []+ for key, values in param_score_dict.items():+ scores = np.array([v[0] for v in values])+ mean = np.average(scores)+ program = values[0][1]+ results.append((key, mean, program))++ # Sort results by the mean+ sorted_results = sorted(results, key=lambda x: x[1], reverse=True)++ # Find the combination with the highest mean, skip fully evaluated ones+ for combination in sorted_results:+ key, mean, program = combination++ if key in fully_evaled_param_combos:+ continue++ print(f"Best Combination: {key} with Mean = {mean}")++ return program, key++ # If no valid program is found, we return the last valid one that we found+ return program, key+++def calculate_last_n_proposed_quality(+ base_program, trial_logs, evaluate, trainset, devset, n+):+ """+ Calculate the average and best quality of the last n programs proposed. This is useful for seeing if our proposals+ are actually 'improving' overtime or not.+ """+ # Get the trials from the last n keys in trial logs+ last_n_trial_nums = list(trial_logs.keys())[-n:]++ # Calculate the average and best score of these trials+ # if num_eval_calls in the trial is less than the trainset, throw a not-implemented error for now+ total_train_score = 0+ best_train_score = 0+ total_dev_score = 0+ best_dev_score = 0+ for trial_num in last_n_trial_nums:+ full_eval = trial_logs[trial_num]["full_eval"]+ if not full_eval:+ raise NotImplementedError(+ "Still need to implement non full eval handling in calculate_last_n_proposed_quality"+ )+ train_score = trial_logs[trial_num]["score"]+ program = base_program.deepcopy()+ program.load(trial_logs[trial_num]["program_path"])++ dev_score = evaluate(program, devset=devset)++ total_train_score += train_score+ total_dev_score += dev_score+ if train_score > best_train_score:+ best_train_score = train_score+ best_dev_score = dev_score++ return best_train_score, total_train_score / n, best_dev_score, total_dev_score / n+++### LOGGING UTILS ###+++def get_task_model_history_for_full_example(+ candidate_program, task_model, devset, evaluate+):+ """Get a full trace of the task model's history for a given candidate program."""+ _ = evaluate(candidate_program, devset=devset[:1])+ _ = task_model.inspect_history(n=len(candidate_program.predictors()))+ return task_model.inspect_history(n=len(candidate_program.predictors()))+++def print_full_program(program):+ """Print out the program's instructions & prefixes for each module."""+ for i, predictor in enumerate(program.predictors()):+ print(f"Predictor {i}")+ print(f"i: {get_signature(predictor).instructions}")+ *_, last_field = get_signature(predictor).fields.values()+ print(f"p: {last_field.json_schema_extra['prefix']}")+ print("\n")+++def save_candidate_program(program, log_dir, trial_num, note=None):+ """Save the candidate program to the log directory."""++ if log_dir is None:+ return++ # Ensure the directory exists+ eval_programs_dir = os.path.join(log_dir, "evaluated_programs")+ os.makedirs(eval_programs_dir, exist_ok=True)++ # Define the save path for the program+ if note:+ save_path = os.path.join(eval_programs_dir, f"program_{trial_num}_{note}")+ else:+ save_path = os.path.join(eval_programs_dir, f"program_{trial_num}")++ # Save the program+ program.save(save_path)++ return save_path+++def save_file_to_log_dir(source_file_path, log_dir):+ if log_dir is None:+ return+ """Save a file to our log directory"""+ if not os.path.exists(log_dir):+ os.makedirs(log_dir)+ destination_file_path = os.path.join(log_dir, os.path.basename(source_file_path))++ # Copy the file+ shutil.copy(source_file_path, destination_file_path)+++def setup_logging(log_dir):+ """Setup logger, which will log our print statements to a txt file at our log_dir for later viewing"""+ if log_dir is None:+ return+ # Create a logger+ logger = logging.getLogger()+ logger.setLevel(logging.DEBUG)++ # Create a file handler that logs debug and higher level messages+ file_handler = logging.FileHandler(f"{log_dir}/logs.txt")+ file_handler.setLevel(logging.DEBUG)+ file_formatter = logging.Formatter("%(asctime)s - %(message)s")+ file_handler.setFormatter(file_formatter)+ logger.addHandler(file_handler)++ # Create a console handler with a higher log level+ console_handler = logging.StreamHandler()+ console_handler.setLevel(logging.INFO)+ console_formatter = logging.Formatter("%(message)s")+ console_handler.setFormatter(console_formatter)+ logger.addHandler(console_handler)+++### OTHER UTILS ###++def get_signature(predictor):+ if hasattr(predictor, "extended_signature"):+ return predictor.extended_signature+ elif hasattr(predictor, "signature"):+ return predictor.signature+ return None+++def set_signature(predictor, updated_signature):+ if hasattr(predictor, "extended_signature"):+ predictor.extended_signature = updated_signature+ elif hasattr(predictor, "signature"):+ predictor.signature = updated_signature+++def create_n_fewshot_demo_sets(+ student,+ num_candidate_sets,+ trainset,+ max_labeled_demos,+ max_bootstrapped_demos,+ metric,+ teacher_settings,+ max_rounds=1,+ labeled_sample=True,+ min_num_samples=1,+ metric_threshold=None,+ teacher=None,+ include_non_bootstrapped=True,+ seed=0,+):+ """+ This function is copied from random_search.py, and creates fewshot examples in the same way that random search does.+ This allows us to take advantage of using the same fewshot examples when we use the same random seed in our optimizers.+ """+ demo_candidates = {}++ # Account for confusing way this is set up, where we add in 3 more candidate sets to the N specified+ num_candidate_sets -= 3++ # Initialize demo_candidates dictionary+ for i, _ in enumerate(student.predictors()):+ demo_candidates[i] = []++ starter_seed = seed+ # Shuffle the trainset with the starter seed+ random.Random(starter_seed).shuffle(trainset)++ # Go through and create each candidate set+ for seed in range(-3, num_candidate_sets):++ trainset2 = list(trainset)++ if seed == -3 and include_non_bootstrapped:+ # zero-shot+ program2 = student.reset_copy()++ elif (+ seed == -2+ and max_labeled_demos > 0+ and include_non_bootstrapped+ ):+ # labels only+ teleprompter = LabeledFewShot(k=max_labeled_demos)+ program2 = teleprompter.compile(+ student, trainset=trainset2, sample=labeled_sample+ )++ elif seed == -1:+ # unshuffled few-shot+ program = BootstrapFewShot(+ metric=metric,+ max_bootstrapped_demos=max_bootstrapped_demos,+ max_labeled_demos=max_labeled_demos,+ teacher_settings=teacher_settings,+ max_rounds=max_rounds,+ )+ program2 = program.compile(student, teacher=teacher, trainset=trainset2)++ else:+ # shuffled few-shot+ random.Random(seed).shuffle(trainset2)+ size = random.Random(seed).randint(min_num_samples, max_bootstrapped_demos)++ teleprompter = BootstrapFewShot(+ metric=metric,+ metric_threshold=metric_threshold,+ max_bootstrapped_demos=size,+ max_labeled_demos=max_labeled_demos,+ teacher_settings=teacher_settings,+ max_rounds=max_rounds,+ )++ program2 = teleprompter.compile(+ student, teacher=teacher, trainset=trainset2+ )++ for i, _ in enumerate(student.predictors()):+ demo_candidates[i].append(program2.predictors()[i].demos)++ return demo_candidates++def create_demo_candidates(program, n, devset, metric, max_bootstrapped_demos, max_labeled_demos, teacher_settings = {}, hard_fewshot=False, seed=0):+ demo_candidates = {}+ # if not program_aware:+ for i in range(n):+ # if i == 0: # Story empty set of demos as default+ # for j, module_p in enumerate(program.predictors()):+ # if j not in demo_candidates.keys():+ # demo_candidates[j] = []+ # demo_candidates[j].append([])+ # else: # Create and set bootstrapped examples+ logging.info(f"Creating basic bootstrap {i+1}/{n}")++ # Create a new basic bootstrap few - shot program .+ rng = random.Random(i+seed)+ shuffled_devset = devset[:] # Create a copy of devset+ rng.shuffle(shuffled_devset) # Shuffle the copy++ if hard_fewshot:+ logging.info(f"Creating hard bootstrap {i+1}/{n}")+ tp = BootstrapHardFewShot(metric = metric, max_bootstrapped_demos=max_bootstrapped_demos, max_labeled_demos=max_labeled_demos, teacher_settings=teacher_settings)+ else:+ logging.info(f"Creating basic bootstrap {i+1}/{n}")+ tp = BootstrapFewShot(metric = metric, max_bootstrapped_demos=max_bootstrapped_demos, max_labeled_demos=max_labeled_demos, teacher_settings=teacher_settings)++ candidate_program = tp.compile(student=program.deepcopy(), trainset=shuffled_devset)++ # Store the candidate demos+ for j, candidate_p in enumerate(candidate_program.predictors()):+ if j not in demo_candidates.keys():+ demo_candidates[j] = []+ demo_candidates[j].append(candidate_p.demos)+ return demo_candidates++def new_getfile(object, _old_getfile=inspect.getfile):+ if not inspect.isclass(object):+ return _old_getfile(object)++ # Lookup by parent module (as in current inspect)+ if hasattr(object, '__module__'):+ object_ = sys.modules.get(object.__module__)+ if hasattr(object_, '__file__'):+ return object_.__file__++ # If parent module is __main__, lookup by methods (NEW)+ for name, member in inspect.getmembers(object):+ if inspect.isfunction(member) and object.__qualname__ + '.' + member.__name__ == member.__qualname__:+ return inspect.getfile(member)+ else:+ raise TypeError('Source for {!r} not found'.format(object))++inspect.getfile = new_getfile++def get_dspy_source_code(module):+ header = []+ base_code = ""+ try:+ base_code = inspect.getsource(type(module))+ except TypeError:+ obj = type(module)+ cell_code = "".join(inspect.linecache.getlines(new_getfile(obj)))+ class_code = extract_symbols(cell_code, obj.__name__)[0][0]+ base_code = str(class_code)++ completed_set = set()+ for attribute in module.__dict__.keys():+ try:+ iterable = iter(getattr(module, attribute))+ except TypeError:+ iterable = [getattr(module, attribute)]++ for item in iterable:+ if item in completed_set:+ continue+ if isinstance(item, Parameter):+ if item.signature is not None and item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig" not in completed_set:+ try:+ header.append(inspect.getsource(item.signature))+ except TypeError:+ header.append(str(item.signature))+ completed_set.add(item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig")+ if isinstance(item, dspy.Module):+ if type(item) not in completed_set:+ header.append(get_dspy_source_code(item))+ completed_set.add(type(item))+ completed_set.add(item)++ return '\n\n'.join(header) + '\n\n' + base_code\ No newline at end of filediff --git a/examples/nli/scone/scone_with_MIPRO.ipynb b/examples/nli/scone/scone_with_MIPRO.ipynbnew file mode 100644index 000000000..9fd3da46d--- /dev/null+++ b/examples/nli/scone/scone_with_MIPRO.ipynb@@ -0,0 +1,3786 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "li3F9kMOqZHz"+ },+ "source": [+ "<img src=\"../../../docs/images/DSPy8.png\" alt=\"DSPy7 Image\" height=\"150\"/>"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "3wEDck3ZqZH0"+ },+ "source": [+ "# Using __<ins>M</ins>ulti-stage <ins>I</ins>nstruction <ins>P</ins>roposal & <ins>O</ins>ptimization (MIPRO)__ in DSPy\n",+ "[](https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "7DzBCQ0UqZH0"+ },+ "source": [+ "### FAQ 🙋\n",+ "#### 1) How does MIPRO work?\n",+ "At a high level, the MIPRO program optimizer works by first __proposing__ candidate fewshot example sets and instructions for each prompt in your program, and then __optimizing__ over these fewshot example sets and instructions as hyperparameters for a specified number of trials. Each trial, the optimizer evaluates different combinations of prompts on a train set, which allows it to learn which combinations yield the best performance.\n",+ "\n",+ "#### 2) How much will MIPRO cost me to run?\n",+ "Note that __this notebook__ is free to run, because all LM calls have been cached. However, when using an optimizer on your own program, here is a breakdown of the upper bound of the number of calls to the task model and prompt model respectively:\n",+ "\n",+ "- **Task model calls**: MIPRO makes up to __O(TxPxM)__ task model calls if you run without minibatching, where T is the number of trials, P is the number of prompts in the program, and M is the size of the train set. This is because the model is evaluating the program on the train set each trial. In practice, this should be lower given that MIPRO tunes poor trials early (ie. it may stop a trial after running on the first 100 or so examples if performance is poor). If you run **with minibatching** you can reduce calls even further to __O(TxPxB)__ where **B** is the minibatch size. Note that every few steps (a parameter you set) MIPRO will also run a full eval over all **M** examples.\n",+ "\n",+ "- **Prompt model calls**: MIPRO makes up to N*P+10+(P+1) prompt model calls, where N is the number of instruction / fewshot example set candidates to generate for each prompt, and P is the number of prompts in the program. The extra 10 calls comes from generating a summary of the data in the training set, which we use in the meta prompt to create better instructions. The extra (P+1) comes from program summarization where the proposer LLM will look at the program code and try to describe what each module does and what the whole program does.\n",+ "\n",+ "#### 3) How should I configure the hyperparameters?\n",+ "We have yet to run full hyperparameter sweeps with MIPRO, but based off of initial experimintation, we'd recommend the following:\n",+ "- __Trial num__: Gains can be seen after about 20-30 trials. However, 100-200 trials can help with adding on additional marginal gains.\n",+ "- __num candidates__: This hyperparameter controls the number of candidate prompts and fewshot example sets that are generated to optimize over. With more trials and less prompts to optimize, we can set n to be higher, as we have more trials to explore different combinations of prompts. If your program has between 2-3 modules and is the `num_trials=30`, we'd recommend ~`n=10`. If n is higher (say `n=100`), then we can go higher to ~`n=15`. If you have a program with only 1 module and are keeping the program 0-shot (ie. no fewshot examples), then `num_trials` should be set to equal `n`, because each trial can explore a new instruction.\n",+ "- __Training set size__: Between 200 and 500 training examples are recommended. Increasing the training set size can help prevent overfitting, but adds to the expense to run.\n",+ "\n",+ "#### 4) What should I do if I want to reduce the cost?\n",+ "You can always update hyperparameters accordingly, such as using a smaller train set, using less trials, or using a program with less modules. You should take advantage of minibatching.\n",+ "Alternatively, one strategy would be to optimize using a cheaper task model (ie. locally hosted Llama-2), as initial experiments have shown that prompts optimized for a smaller model also transfer to working well on a larger model.\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "SgTc-CutqZH1"+ },+ "source": [+ "### 0] Setup"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "5Vo4Tb9srSow"+ },+ "source": [+ "First, we will install __DSPy__ if it's not there already. We'll also __load in the cached requests__ for this tasks, so that we don't actually need to call any LMs for this notebook. We'll also load in our pre optimized program from hugging face to inspect later."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {+ "colab": {+ "base_uri": "https://localhost:8080/"+ },+ "id": "JpijP_d7qZH2",+ "outputId": "c641a4b1-05f5-45cc-d715-4347c526b576"+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/opt/miniconda3/envs/opt-prompt/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ }+ ],+ "source": [+ "%load_ext autoreload\n",+ "%autoreload 2\n",+ "\n",+ "import sys\n",+ "import os\n",+ "\n",+ "try: # When on google Colab, let's clone the notebook so we download the cache.\n",+ " import google.colab\n",+ " repo_path = 'dspy'\n",+ "\n",+ " !git -C $repo_path pull origin || git clone https://github.com/stanfordnlp/dspy $repo_path\n",+ "except:\n",+ " repo_path = '.'\n",+ "\n",+ "if repo_path not in sys.path:\n",+ " sys.path.append(repo_path)\n",+ "\n",+ "\n",+ "import pkg_resources # Install the package if it's not installed\n",+ "if not \"dspy-ai\" in {pkg.key for pkg in pkg_resources.working_set}:\n",+ " !pip install -U pip\n",+ " !pip install dspy-ai\n",+ " !pip install openai~=0.28.1\n",+ " !pip install -e $repo_path\n",+ " !pip install --upgrade cloudpickle==3.0.0\n",+ "\n",+ "from huggingface_hub import hf_hub_download\n",+ "import zipfile\n",+ "import os\n",+ "\n",+ "repo_id = 'kopsahlong/DSPy_MIPRO_notebook_cache'\n",+ "cache_file_path = hf_hub_download(repo_id=repo_id, filename='MIPRO_notebook_cache.zip')\n",+ "compiled_program_file_path = hf_hub_download(repo_id=repo_id, filename='compiled_program.pickle')\n",+ "with zipfile.ZipFile(cache_file_path, 'r') as zip_ref:\n",+ " zip_ref.extractall(\".\")\n",+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = f\"{os.getcwd()}/MIPRO_notebook_cache\"\n",+ "\n",+ "import dspy\n",+ "import pandas as pd\n",+ "import glob\n",+ "import random\n",+ "from dspy.evaluate import Evaluate"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "U-DaFCBvqZH2"+ },+ "source": [+ "We will also specify the __prompt LM model__ (in this case GPT 3.5), the __task LM model__ (Llama 13B) and the retrieval model we'll be using for our task (a HotPotQA multihop retrieval task)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {+ "id": "UHWzGRVgqZH2"+ },+ "outputs": [],+ "source": [+ "import openai\n",+ "\n",+ "### NOTE: if you'd like to run this code without a cache, you can remove these lines to configure your OPEN AI key ###\n",+ "# os.environ['OPENAI_API_KEY'] = \"TODO: ADD YOUR OPEN AI KEY HERE\"\n",+ "# openai.api_key = os.environ.get('OPENAI_API_KEY')\n",+ "# openai.api_base = \"https://api.openai.com/v1\"\n",+ "\n",+ "prompt_model_name = \"gpt-3.5-turbo-1106\"\n",+ "task_model_name = \"meta-llama/Meta-Llama-3-8B\"\n",+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",+ "\n",+ "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=1000, stop=[\"\\n\\n\", \"\\n---\"])\n",+ "task_model = dspy.HFClientVLLM(\n",+ " model=task_model_name,\n",+ " port=7410,\n",+ " url=[\"http://future-hgx-2:7500\", \"http://future-hgx-2:7501\", \"http://future-hgx-2:7502\", \"http://future-hgx-2:7503\", \"http://future-hgx-2:7504\", \"http://future-hgx-2:7505\", \"http://future-hgx-2:7506\", \"http://future-hgx-2:7507\"],\n",+ " max_tokens=1000,\n",+ " stop=[\"\\n\\n\", \"\\n---\", \"assistant\"],\n",+ ")\n",+ "\n",+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",+ "\n",+ "dspy.settings.configure(rm=colbertv2, lm=task_model)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "BFoPwDrUqZH2"+ },+ "source": [+ "### 1] Define Task"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "s4Cyb1JtqZH2"+ },+ "source": [+ "Here, we'll define the program that we'd like to run, which is a multihop [...] (we can say that it was loosely inspired by a certain paper). We additionally load in the data, and define how we'd like to evaluate this task."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "fatal: destination path 'ScoNe' already exists and is not an empty directory.\n"+ ]+ }+ ],+ "source": [+ "!git clone https://github.com/selenashe/ScoNe.git"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "class ScoNeSignature(dspy.Signature):\n",+ " (\"\"\"context, question -> answer\"\"\")\n",+ "\n",+ " context = dspy.InputField()\n",+ " question = dspy.InputField()\n",+ " answer = dspy.OutputField(desc=\"Yes or No\")\n",+ "\n",+ "class ScoNeCoT(dspy.Module):\n",+ " def __init__(self):\n",+ " super().__init__()\n",+ " self.generate_answer = dspy.ChainOfThought(ScoNeSignature)\n",+ "\n",+ " def forward(self, context, question):\n",+ " return self.generate_answer(context=context, question=question)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {+ "colab": {+ "base_uri": "https://localhost:8080/",+ "height": 340,+ "referenced_widgets": [+ "e16044d880174c49af557bd12789493f",+ "531c380db44a404ebb168648ea77c3f4",+ "e8a46c6ac8a54fdbb44b2c8914552e64",+ "81cbe1842465400cba02a46309d98064",+ "f8ada809ecdb417ebc8214d860dd6552",+ "74d58314b1c449e0b7dec3bda8b653c2",+ "8b7b2b9489ae49049befae848d0fb1a1",+ "2d47a6a66054438d8e9a3c5e5767c056",+ "3b57d0a9768f4befb7509581289035ad",+ "134fa83980e142bf82d5b97cfc10da69",+ "35e4e99036894a2ba37d9ba23581a8ff",+ "835a1e675186490692e336da943d635d",+ "95bcef02eb794979b7873b81021e4b40",+ "d84324c4c3dc40fea04cc0df1539b4d8",+ "a6213bcdbcb24b1694204e6baeb763d8",+ "bc423d0878154adf940062660a8315ad",+ "0a1744c363f640b5b8939f32f6e72922",+ "bbd2db64988147f1a4f8856d890bb4c7",+ "ba2b4d7ecdbc4512acf180d8a0d602e8",+ "1bb155669e2a441d8992030e8aaa84a9",+ "43ed7af1d9c84ac8a6ec2195e48e60eb",+ "ac6822762e6b46bd862d6f923aeb4437",+ "348bb4fff900492cba8333156626a947",+ "697eaeb1a004493a9260a3a89671a9ca",+ "8a64950ea896468da3d10f46e9718ec8",+ "7b0b45020d0f45288e2f0e4ceff22524",+ "ad21e20d1f1b4b6ea74fdab03af8acdb",+ "1aab1d48c6124526bee6c74892ecd953",+ "598c6f37331e448889b175393a00deff",+ "20d8df0ee0a4442582686846757bcc7f",+ "fa4e50aaf53d4edfb9ca3046cbada2db",+ "96ad4454539145aea6549995344160e3",+ "acc3573e48f14dffb635f8632c6f6e74",+ "6c0432c6cd8f4cae9aeb8c2870b39922",+ "99b8f3882032404990f2b453111a63ba",+ "6c21df0657ce486381ba2310c7fa0029",+ "a20ac344a32340c785cd162fd0eb55b5",+ "6ba50c3ec9e542d5a8d2f900fbcb8689",+ "a298045042014c88939a9fe785fccda8",+ "84bed63c6597486ca6c39b9c0c4d20bd",+ "2a75d45acfb44463af974acb7a1b0d8e",+ "ae814b8e55454e0aadc48f7e595575ee",+ "ff4bcec9c18e4f04b1f7898daaffeb1a",+ "e9c64a790d684c9eaf7f49eea003f43c",+ "52f1714412df4ec78f6ff7d0f8a69862",+ "988055b44cf9411e8d45d8a4185fef07",+ "59be08d44c824d76b8525df14191d569",+ "6fde64f36dd84d8eae670efe95e2313a",+ "66e6502a463145daa440e967d8bdfdca",+ "29cab51fec0b4dacaa8f829ff217c839",+ "6ecbf23579324112981d4bce0c0ce369",+ "a105ef4f1f754c44b7bc0ec8edf0c0cc",+ "1c7d71e42c8c494f867b30d781beb681",+ "2f1e652cfa514054abfda794bdfa61a5",+ "b17f4731e8e44858889114c52b8f3dd4",+ "d50f5eafe90c4684bbdd96569b8ff247",+ "9f07fa213de247dabcd3f4213d35a97e",+ "782af098d37d4543a4a01ecfed4e5da2",+ "037b10adf9724e058c6468f4ca74ed86",+ "c7d10443100e49d28266aef9f644ed47",+ "d6a78aabcca74314a5b4bb98f350693a",+ "954f3cfcb5d44dddbf1d4db9e99c0d70",+ "62d8196169bb4a76a94c16d6f1ca61a6",+ "0a16c7f37b9a4bbe9bfe7e737ea62801",+ "942897aa22214745adee56d2c54447e9",+ "0b49910850f3445abe63b6bc7ac18412",+ "12a9763cd97742f4ab80c0494a398ca6",+ "9d4f14862b704d9bb53d9e74365d14ac",+ "56989c76e1534cfd8c9b0da93b3b8bf8",+ "f120c447645446e4a04791b97ce9ae93",+ "6a5d5ea44c0d4e4384b8956b48c34f14",+ "e3dd508496f44171b0d91aca0e8b16a5",+ "dc8077859eec4261a28d708ab06a1008",+ "9a733957f8fc446fbde053ba87289c71",+ "287c9f993cd44039923fdc122cd9e040",+ "5cfebfd6308349038f9cd7ad5bc00fe5",+ "56b80fc45dd247deadceffe17557f0f9",+ "7d80528c4b1c48318756230b0174923c",+ "398d7d36bbd041fe81ec633e973fc504",+ "d01ca0dd46ee4a4daeeee92214bc07d1",+ "6028a5e08f8641f5b8e8182e50f55419",+ "567646442eb940b09456328d49248945",+ "41961130caaf4537a4c1793574c785d7",+ "e9935b60c48d46469904492e20f9c2e8",+ "f048b50740e94bd8805c142eac1673b6",+ "a5effa5d67534fb4be2e3320c1fb2b9f",+ "09b3f2c456af41cba9264dbb9a724027",+ "e2f3a3377ad64a4cb9f3a42c1ac97344",+ "8e396041a4fd4e6db02410a09b7556c7",+ "4cd1dc71c80b401da19ed52ef5148d60",+ "1f23a95e292249a19055f70cda2622a3",+ "69412c7605ec48859baef6f31c91c520",+ "ee64a46b61454949ade4177c059bcf61",+ "de6507e9f45741218bf31a9af728b2bf",+ "38abb8f460d24c27a66c54bb0417f8ce",+ "33267de625db44c2a63d7c7d412a7e61",+ "19fe27a0df5a4d39873dd1031904405c",+ "dba7ce7c756d468b94d0bc8f4815ab6f",+ "926cbb119ea745cf9e344c0e50b75f62"+ ]+ },+ "id": "hiVgd3N7qZH3",+ "outputId": "81b97d83-7c5f-4763-d104-3ad320425a2a"+ },+ "outputs": [],+ "source": [+ "def load_scone(dirname):\n",+ " dfs = []\n",+ " for filename in glob.glob(dirname + \"/*.csv\"):\n",+ " df = pd.read_csv(filename, index_col=0)\n",+ " df['category'] = os.path.basename(filename).replace(\".csv\", \"\")\n",+ " dfs.append(df)\n",+ " data_df = pd.concat(dfs)\n",+ "\n",+ " def as_example(row):\n",+ " # The 'one_scoped' file is from an earlier dataset, MoNLI, and\n",+ " # so is formatted a bit differently:\n",+ " suffix = '' if row['category'] == 'one_scoped' else '_edited'\n",+ " # Reformat the hypothesis to be an embedded clause in a question:\n",+ " hkey = 'sentence2' + suffix\n",+ " question = row[hkey][0].lower() + row[hkey][1: ].strip(\".\")\n",+ " question = f\"Can we logically conclude for sure that {question}?\"\n",+ " # Binary task formulation:\n",+ " label = \"Yes\" if row['gold_label' + suffix] == 'entailment' else \"No\"\n",+ " return dspy.Example({\n",+ " \"context\": row['sentence1' + suffix],\n",+ " \"question\": question,\n",+ " \"answer\": label,\n",+ " \"category\": row['category'],\n",+ " }).with_inputs(\"context\", \"question\")\n",+ " return list(data_df.apply(as_example, axis=1).values)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Load and configure the datasets.\n",+ "all_train = load_scone(\"ScoNe/scone_nli/train\")\n",+ "\n",+ "random.seed(1)\n",+ "random.shuffle(all_train)\n",+ "\n",+ "# 1000 random train, 500 random dev:\n",+ "trainset, valset, testset = all_train[: 200], all_train[200: 400], all_train[400: 600]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Set up metrics\n",+ "NUM_THREADS = 10\n",+ "metric = dspy.evaluate.answer_exact_match\n",+ "\n",+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",+ "evaluate = Evaluate(devset=valset, metric=metric, **kwargs)\n",+ "\n",+ "program = ScoNeCoT()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "hRGld1C1qZH3"+ },+ "source": [+ "### 2] Baseline Evaluation\n",+ "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __58.0%__ on our trainset, __49.5%__ on our valset, and __55.0%__ on our testset."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {+ "colab": {+ "base_uri": "https://localhost:8080/"+ },+ "id": "MU2aHQBTqZH3",+ "outputId": "786ba2f2-ae0a-4c68-b602-d601fb5a5aa5"+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 116 / 200 (58.0): 100%|██████████| 200/200 [00:00<00:00, 866.98it/s]\n",+ "Average Metric: 99 / 200 (49.5): 100%|██████████| 200/200 [00:00<00:00, 630.52it/s]\n",+ "Average Metric: 110 / 200 (55.0): 100%|██████████| 200/200 [00:00<00:00, 957.81it/s]\n"+ ]+ }+ ],+ "source": [+ "baseline_train_score = evaluate(program,devset=trainset)\n",+ "baseline_eval_score = evaluate(program, devset=valset)\n",+ "baseline_test_score = evaluate(program, devset=testset)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "cjCoL27yqZH3"+ },+ "source": [+ "### 3] Optimizing with MIPRO"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "eEGOKjXprc7z"+ },+ "source": [+ "Now let's get into the key method in this notebook - optimizing our program with MIPRO!"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "G04L5j9iqZH3"+ },+ "source": [+ "#### 3a] Compile Program\n",+ "First, we'll get our optimized program. By default, we set `LOAD_PRECOMPILED_PROGRAM` to `True`, so that you can quickly access a program we've precompiled for you. However, if you wish to optimize yourself, `LOAD_PRECOMPILED_PROGRAM` can be set to `False` (though please note that this will require adding in your own LM API keys in the __Setup__ section above).\n",+ "\n",+ "MIPRO only needs a metric, DSPy module, and training set to see huge gains on your task! You can instantiate a MIPRO Optimizer and compile in just two lines:\n",+ "```python\n",+ "teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature)\n",+ "compiled_program = teleprompter.compile(program, trainset=trainset, devset=devset, num_trials=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)\n",+ "```"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {+ "id": "NVfMJ_FpBlSI"+ },+ "outputs": [],+ "source": [+ "import cloudpickle as pickle\n",+ "from dspy.teleprompt import MIPRO\n",+ "\n",+ "LOAD_PRECOMPILED_PROGRAM = False\n",+ "\n",+ "# By default, we will load the precompiled program\n",+ "if LOAD_PRECOMPILED_PROGRAM:\n",+ " # Load a our precompiled program\n",+ " with open(compiled_program_file_path, 'rb') as file:\n",+ " # Load the data from the file\n",+ " compiled_program = pickle.load(file)\n",+ "# Otherwise, if desired, the program can be compiled from scratch\n",+ "else:\n",+ " # Define hyperparameters:\n",+ " N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",+ " batches = 30 # The number of optimization batches to be run (we will test out a new combination of instructions and fewshot examples in each trial)\n",+ " temperature = 1.0 # The temperature configured for generating new instructions\n",+ "\n",+ " # Compile\n",+ " eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",+ " teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n",+ " compiled_program = teleprompter.compile(program, trainset=trainset, valset=valset, num_batches=batches, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "uqVVnaEBqZH3"+ },+ "source": [+ "#### 3b] Evaluate optimized program\n",+ "Now, we evaluate our program that has been optimized with MIPRO. We see that performance on train and dev have improved by __+13.0pt__, __+15.5pt__, and __+15.5pt__ respectively!"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {+ "id": "VvnBp7huqZH3"+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 142 / 200 (71.0): 100%|██████████| 200/200 [00:00<00:00, 878.26it/s]\n",+ "Average Metric: 130 / 200 (65.0): 100%|██████████| 200/200 [00:00<00:00, 602.80it/s]\n",+ "Average Metric: 141 / 200 (70.5): 100%|██████████| 200/200 [00:00<00:00, 854.88it/s]\n"+ ]+ }+ ],+ "source": [+ "bayesian_train_score = evaluate(compiled_program, devset=trainset)\n",+ "bayesian_val_score = evaluate(compiled_program, devset=valset)\n",+ "bayesian_test_score = evaluate(compiled_program, devset=testset)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "j3UWn_UnqZH4"+ },+ "source": [+ "#### 3c] Visualizing scores & prompts over trials"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "pBYTLTwWqZH4"+ },+ "source": [+ "Now, let's take a look at how this optimization looked over the course of each trial. We see that, in general, __performance increases overtime__, until it saturates after ~trial #13."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {+ "id": "rtMUNeicqZH4"+ },+ "outputs": [+ {+ "data": {+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAE8CAYAAACrYErbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKQklEQVR4nO3deVhU5fs/8PcwsskqggKySiioqIFLqOASimsqqLlU7maiiZSllQpmbn0/5pLpJzM1zSUN/VTuSyq4i3sSqaGYIioug6KAzPP7w99MDsMgMwzMwLxf1zXXxTznzDP3fc7M3Jz1kQghBIiIiEiNmaEDICIiMlYskkRERBqwSBIREWnAIklERKQBiyQREZEGLJJEREQasEgSERFpwCJJRESkAYskERGRBiySVKJ27dqhXbt2Or1WIpEgPj5er/GYMm2Wp0QiwdixY8s3oAq2cuVKSCQSXL161dChkAlhkaziJBJJqR779+83WIx37tzB+PHjERAQAGtra9SqVQstWrTAxx9/jEePHhksLmN3+PBhxMfH48GDB3rt9+rVqyqfDalUCi8vL/Tu3RtnzpzR63tVJsnJyejSpQvq1KkDKysreHl5oUePHli7dq1yntzcXMTHx5fp+1Re65V0U83QAVD5Wr16tcrzH374Abt371ZrDwwMLPb1u3btKrfYAODevXto1qwZZDIZhg0bhoCAAGRnZ+PcuXNYsmQJ3nvvPdja2pZrDJXFkydPUK3av1/Zw4cPIyEhAUOGDIGjo6Pe32/AgAHo2rUrCgsLkZqaiiVLlmD79u04evQomjZtqvf3e5m3334b/fv3h6WlZYW/98aNG/Hmm2+iadOmGD9+PGrUqIH09HQcPHgQy5Ytw8CBAwE8L5IJCQkAoPMemPJer6QdFskq7q233lJ5fvToUezevVutvajc3FxUr14dFhYW5Rkeli9fjoyMDBw6dAitWrVSmSaTycr9/V/0+PFj2NjYVNj7acvKyqpC3y84OFjlc9K6dWu88cYbWLJkCf773/8W+5ryXIZSqRRSqbRc+n6Z+Ph4NGjQAEePHlX7TN6+fdsgMVHF4O5WQrt27dCoUSOkpKQgPDwc1atXxyeffKKc9uJ/xPn5+Zg6dSpCQkLg4OAAGxsbhIWF4ffff9fpva9cuQKpVIrXXntNbZq9vb1aYTh27Bi6du2KGjVqwMbGBo0bN8aCBQtU5tm3bx/CwsJgY2MDR0dH9OzZE6mpqSrzxMfHQyKR4OLFixg4cCBq1KiBNm3aKKevWbMGISEhsLa2hpOTE/r374/r16+r9HHp0iVER0fD1dUVVlZW8PDwQP/+/fHw4UON+S5cuBBSqVRlV9p//vMfSCQSxMXFKdsKCwthZ2eHjz/+WNn24jHJ+Ph4TJw4EQDg6+ur3DVa9Hjdli1b0KhRI1haWqJhw4bYsWOHxthepkOHDgCA9PR0AP8eIzxw4ADGjBmDWrVqwcPDAwAwZMgQ+Pj4qPWhWO4vUhw/fVmsxR2T9PHxQffu3ZGcnIwWLVrAysoKdevWxQ8//KD23ufOnUPbtm1hbW0NDw8PzJgxAytWrCjVcc4rV66gefPmxf7TVqtWLQDPd1O7uLgAABISEpTrRLHOzp07hyFDhqBu3bqwsrKCq6srhg0bhuzsbJXlo2m9KnaDr1y5Ui2Goserc3JyEBsbCx8fH1haWqJWrVro2LEjTp06VWKepI5bkgQAyM7ORpcuXdC/f3+89dZbqF27drHzyWQyfPfddxgwYABGjhyJnJwcLF++HJGRkTh+/LjWu+G8vb1RWFiI1atXY/DgwSXOu3v3bnTv3h1ubm4YP348XF1dkZqait9++w3jx48HAOzZswddunRB3bp1ER8fjydPnmDRokVo3bo1Tp06pfbD3bdvX/j7+2PmzJlQjBr3xRdfYMqUKejXrx9GjBiBO3fuYNGiRQgPD8fp06fh6OiI/Px8REZGIi8vD+PGjYOrqytu3LiB3377DQ8ePICDg0OxOYSFhUEulyM5ORndu3cHACQlJcHMzAxJSUnK+U6fPo1Hjx4hPDy82H6ioqLw119/Yd26dfjqq6/g7OwMAMofaeD5MbTExESMGTMGdnZ2WLhwIaKjo5GRkYGaNWuWuKyLc+XKFQBQe+2YMWPg4uKCqVOn4vHjx1r3W9ZYL1++jD59+mD48OEYPHgwvv/+ewwZMgQhISFo2LAhAODGjRto3749JBIJJk+eDBsbG3z33Xel3nXr7e2NvXv34p9//lH+I1CUi4uL8hBB7969ERUVBQBo3LgxgOef37///htDhw6Fq6sr/vjjD3z77bf4448/cPToUUgkkhLX6507d0oVKwCMHj0amzZtwtixY9GgQQNkZ2cjOTkZqampCA4OLnU/BECQSYmJiRFFV3vbtm0FALF06VK1+du2bSvatm2rfP7s2TORl5enMs/9+/dF7dq1xbBhw1TaAYhp06aVGM+tW7eEi4uLACACAgLE6NGjxdq1a8WDBw9U5nv27Jnw9fUV3t7e4v79+yrT5HK58u+mTZuKWrVqiezsbGXb2bNnhZmZmXjnnXeUbdOmTRMAxIABA1T6unr1qpBKpeKLL75QaT9//ryoVq2asv306dMCgNi4cWOJ+RVVWFgo7O3txUcffaSMvWbNmqJv375CKpWKnJwcIYQQ8+bNE2ZmZiq5Fl2eX375pQAg0tPT1d4HgLCwsBCXL19WWQ4AxKJFi0qMMT09XQAQCQkJ4s6dO+LWrVti//794tVXXxUAxM8//yyEEGLFihUCgGjTpo149uyZSh+DBw8W3t7ean0rlrsusSre78V8vb29BQBx8OBBZdvt27eFpaWl+OCDD5Rt48aNExKJRJw+fVrZlp2dLZycnDQuwxctX75cGWf79u3FlClTRFJSkigsLFSZ786dOxo/97m5uWpt69atU4tf03pVrJcVK1ao9VP0PR0cHERMTEyJOVHpcHcrAQAsLS0xdOjQl84nlUqVu5zkcjnu3buHZ8+eoVmzZjrtyqlduzbOnj2L0aNH4/79+1i6dCkGDhyIWrVq4fPPP1du3Z0+fRrp6emIjY1VO5lBsfsuMzMTZ86cwZAhQ+Dk5KSc3rhxY3Ts2BHbtm1Te//Ro0erPE9MTIRcLke/fv1w9+5d5cPV1RX+/v7K3cqKLcWdO3ciNze31PmamZmhVatWOHjwIAAgNTUV2dnZmDRpEoQQOHLkCIDnW5eNGjUq04kbERER8PPzUz5v3Lgx7O3t8ffff5fq9dOmTYOLiwtcXV3Rrl07XLlyBXPmzFFuISmMHDmyzMcKyxJrgwYNEBYWpnzu4uKC+vXrq7x2x44dCA0NVdnT4eTkhEGDBpUqvmHDhmHHjh1o164dkpOT8fnnnyMsLAz+/v44fPhwqfqwtrZW/v306VPcvXtXeZhB37tBHR0dcezYMdy8eVOv/ZoiFkkCANSpU6fUJ8msWrUKjRs3hpWVFWrWrAkXFxds3bq1xGNxJXFzc8OSJUuQmZmJtLQ0LFy4ULn7bvny5QD+3dXXqFEjjf1cu3YNAFC/fn21aYGBgbh7967a7kBfX1+V55cuXYIQAv7+/nBxcVF5pKamKk/S8PX1RVxcHL777js4OzsjMjISixcvLtUyCAsLQ0pKCp48eYKkpCS4ubkhODgYTZo0Ue5yTU5OVvnh14WXl5daW40aNXD//v1SvX7UqFHYvXs39u7di5SUFNy+fRsfffSR2nxFl2FFx1qa1167dg2vvPKK2nzFtWkSGRmJnTt34sGDBzh48CBiYmJw7do1dO/evVQn79y7dw/jx49H7dq1YW1tDRcXF+Wy0/W7o8ncuXNx4cIFeHp6okWLFoiPjy/1P0ekisckCYDqf7klWbNmDYYMGYJevXph4sSJqFWrFqRSKWbNmqUsZLqSSCSoV68e6tWrh27dusHf3x8//vgjRowYUaZ+S1I0b7lcDolEgu3btxe7dfTi5Sj/+c9/MGTIEPzvf//Drl278P7772PWrFk4evSoxuNWANCmTRsUFBTgyJEjSEpKUhbDsLAwJCUl4c8//8SdO3fKXCQ1bd0pts5fxt/fHxERES+dr7jPTtGTcxQKCwuLbS9LrGXNU1vVq1dHWFgYwsLC4OzsjISEBGzfvv2lx9T79euHw4cPY+LEiWjatClsbW0hl8vRuXNnyOXyl76vNsu0X79+CAsLw+bNm7Fr1y58+eWXmDNnDhITE9GlS5fSJUoAWCRJS5s2bULdunWRmJio8qWdNm2aXt+nbt26qFGjBjIzMwFAuSvuwoULGn+4vb29AQBpaWlq0/788084Ozu/9PIEPz8/CCHg6+uLevXqvTTOoKAgBAUF4bPPPsPhw4fRunVrLF26FDNmzND4mhYtWsDCwgJJSUlISkpSns0YHh6OZcuWYe/evcrnJdH0o2kMatSoUezF8Iqt/Yrm7e2Ny5cvq7UX16aNZs2aAYDyc6ppndy/fx979+5FQkICpk6dqmy/dOmS2rya+qhRowYAqC1XTcvUzc0NY8aMwZgxY3D79m0EBwfjiy++YJHUEne3klYU/7W/+F/6sWPHlMfStHXs2LFiz4g8fvw4srOzlbtOg4OD4evri/nz56v9SChicXNzQ9OmTbFq1SqVeS5cuIBdu3aha9euL40nKioKUqkUCQkJalsiQgjl6foymQzPnj1TmR4UFAQzMzPk5eWV+B5WVlZo3rw51q1bh4yMDJUtySdPnmDhwoXw8/ODm5tbif0oCr4x3pnFz88PDx8+xLlz55RtmZmZ2Lx5s0HiiYyMxJEjR1TuGHTv3j38+OOPpXq94h+XohTHuRWf0+rVqwNQXyfFfW8AYP78+Wp9alqv9vb2cHZ2Vh7PVvjmm29UnhcWFqrtvq1Vqxbc3d1f+tkkddySJK10794diYmJ6N27N7p164b09HQsXboUDRo00OkWcqtXr8aPP/6I3r17IyQkBBYWFkhNTcX3338PKysr5fWaZmZmWLJkCXr06IGmTZti6NChcHNzw59//ok//vgDO3fuBAB8+eWX6NKlC0JDQzF8+HDlJSAODg6luu+pn58fZsyYgcmTJ+Pq1avo1asX7OzskJ6ejs2bN2PUqFH48MMPsW/fPowdOxZ9+/ZFvXr18OzZM6xevRpSqRTR0dEvfZ+wsDDMnj0bDg4OCAoKAvD8h6x+/fpIS0vDkCFDXtpHSEgIAODTTz9F//79YW5ujh49ehjFDRH69++Pjz/+GL1798b777+P3NxcLFmyBPXq1TPItXofffQR1qxZg44dO2LcuHHKS0C8vLxw7969l26V9+zZE76+vujRowf8/Pzw+PFj7NmzB7/++iuaN2+OHj16AHi+67lBgwbYsGED6tWrBycnJzRq1AiNGjVCeHg45s6di4KCAtSpUwe7du1SXnP6opLW64gRIzB79myMGDECzZo1w8GDB/HXX3+pvD4nJwceHh7o06cPmjRpAltbW+zZswcnTpzAf/7zHz0tURNimJNqyVA0XQLSsGHDYucvegmIXC4XM2fOFN7e3sLS0lK8+uqr4rfffiv2lH+U4hKQc+fOiYkTJ4rg4GDh5OQkqlWrJtzc3ETfvn3FqVOn1OZPTk4WHTt2FHZ2dsLGxkY0btxY7ZKGPXv2iNatWwtra2thb28vevToIS5evKgyj+JShDt37hQb188//yzatGkjbGxshI2NjQgICBAxMTEiLS1NCCHE33//LYYNGyb8/PyElZWVcHJyEu3btxd79uwpMV+FrVu3CgCiS5cuKu0jRowQAMTy5cvVXlPc8vz8889FnTp1hJmZmcplAwCKvQTA29tbDB48uMTYFJcafPnllyXOp7gk48SJE8VO37Vrl2jUqJGwsLAQ9evXF2vWrNF4CUhpYtV0CUi3bt3UXlv0cyvE88t2wsLChKWlpfDw8BCzZs0SCxcuFADErVu3Ssx13bp1on///sLPz09YW1sLKysr0aBBA/Hpp58KmUymMu/hw4dFSEiIsLCwUFln//zzj+jdu7dwdHQUDg4Oom/fvuLmzZtardfc3FwxfPhw4eDgIOzs7ES/fv3E7du3VfrIy8sTEydOFE2aNFF+T5o0aSK++eabEnOk4kmEKKej20RERi42Nhb//e9/8ejRI4Pd8o6MG49JEpFJePLkicrz7OxsrF69Gm3atGGBJI14TJKITEJoaCjatWuHwMBAZGVlYfny5ZDJZJgyZYqhQyMjxiJJRCaha9eu2LRpE7799ltIJBIEBwdj+fLlL73Uhkwbj0kSERFpwGOSREREGhi0SCrGPPP29oa1tTVatWqFEydOKKcLITB16lS4ubnB2toaERERxd6hgoiIqDwY9JjkiBEjcOHCBaxevRru7u5Ys2YNIiIicPHiRdSpUwdz587FwoULsWrVKvj6+mLKlCmIjIzExYsXSz1Ku1wux82bN2FnZ2fUt/EiIqLyI4RATk4O3N3dYWamxfahoS7QzM3NFVKpVPz2228q7cHBweLTTz8VcrlcuLq6qlzQ/ODBA2FpaSnWrVtX6ve5fv26AMAHH3zwwQcf4vr161rVKoNtST579gyFhYVqW4TW1tZITk5Geno6bt26pXIzawcHB7Rs2RJHjhxB//79i+03Ly9P5f6E4v+fl5Seng47Ozud4y0oKMDvv/+O9u3bw9zcXOd+KgtTyxdgzqaQs6nlCzBnRc45OTnw9fXVug4Y9OzWVq1awcLCAmvXrkXt2rWxbt06DB48GK+88gpWrFiB1q1b4+bNmyo3eu7Xrx8kEgk2bNhQbJ/x8fFISEhQa1+7dq3y5sNERGRacnNzMXDgQDx8+BD29valfp1Bj0muXr0aw4YNQ506dSCVShEcHIwBAwYgJSVF5z4nT56MuLg45XOZTAZPT0906tRJqwVTVEFBAXbv3o2OHTuaxH9jppYvwJxNIWdTyxdgzoqcZTKZTn0ZtEj6+fnhwIEDePz4MWQyGdzc3PDmm2+ibt26cHV1BQBkZWWpbElmZWWhadOmGvu0tLSEpaWlWru5ublePiD66qeyMLV8AeZsCkwtX4A565q7UVwnaWNjAzc3N9y/fx87d+5UDkvj6uqqMo6bTCbDsWPHEBoaasBoiYj0o1BeiP1X92Pd+XXYf3U/CuWFhg6JijDoluTOnTshhED9+vVx+fJlTJw4EQEBARg6dCgkEgliY2MxY8YM+Pv7Ky8BcXd3R69evfQahxBCeSKRJgUFBahWrRqePn1a4nxVhanlC5RvzlKpFNWqVeNlSKSUmJqI8TvG4x/ZP8o2D3sPLOi8AFGBUQaMjF5k0CL58OFDTJ48Gf/88w+cnJwQHR2NL774QrlZ/NFHH+Hx48cYNWoUHjx4gDZt2mDHjh2lvkayNPLz85GZmYnc3NwS5xNCwNXVFdevXzeJHzpTyxco/5yrV68ONzc3WFhY6L1vqlwSUxPR56c+EFA9b/KG7Ab6/NQHm/ptYqE0EgYtkv369UO/fv00TpdIJJg+fTqmT59eLu8vl8uRnp4OqVQKd3d3WFhYaPxxlMvlePToEWxtbbW7ELWSMrV8gfLLWQiB/Px83LlzB+np6fD39zeZZUrqCuWFGL9jvFqBBAABAQkkiN0Ri571e0JqxiG8DM2kRwHJz8+HXC6Hp6fnSy8PkcvlyM/Ph5WVlUn8wJlavkD55mxtbQ1zc3Ncu3ZN+R5kmpIyklR2sRYlIHBddh1JGUlo59Ou4gKjYpnGr99LmEoRIMPi54wAIDMnU6/zUfnit5aIqAK52bm9fCYt5qPyxSJJRFSBwrzC4GHvAQmKP/9BAgk87T0R5hVWwZFRcVgkyegcOnQIQUFBMDc3L/XlPvHx8SXeZEIXEokEW7ZsKfX85REDVT1SMykWdF4AAGqFUvF8fuf5PGnHSLBIVkJ37tzBe++9By8vL1haWsLV1RWRkZE4dOiQoUPTi7i4ODRt2hTp6elYuXJlmfuLj4+HRCIp8VGczMxMdOnSpczvT1RUVGAUNvXbhDr2dVTaPew9ePmHkTHps1v1pVBeiKSMJGTmZMLNzg1hXmHl+l9gdHQ08vPzsWrVKtStWxdZWVnYu3cvsrOzy+09K9KVK1cwevRoeHh46KW/Dz/8EKNHj1Y+b968OUaNGoWRI0cWO7/i7FPFrRGJykNUYBR61u9Zob8dpD1uSZZRYmoifBb4oP2q9hiYOBDtV7WHzwIfJKYmlsv7PXjwAElJSZgzZw7at28Pb29vtGjRApMnT8Ybb7wBALh69SokEgnOnDmj8jqJRIL9+/cr2/744w90794d9vb2sLOzQ1hYGK5cuaKcvmbNGgQFBcHS0hJubm4YO3asSn8jRoyAi4sL7O3t0aFDB5w9e1Y5/ezZs2jfvj3s7Oxgb2+PkJAQnDx5EgBw7do19OjRAzVq1ICNjQ0aNmyIbdu2KePOzs7GsGHDIJFIsHLlSqxcuRKOjo4qy2HLli2lvuDf1tYWrq6uyodUKoWdnZ3yef/+/TF27FhMmDABfn5+yq3HortbP/74Y9SrVw/Vq1dH3bp1MWXKFBQUFJQqBqLiSM2kaOfTDgOCBqCdTzsWSCPEIlkGirtmFL3mSXHXjPIolLa2trC1tcWWLVtUxs3U1o0bNxAeHg5LS0vs27cPKSkpGDZsGJ49ewYAWLJkCSZOnIiRI0fi/Pnz+OWXX/DKK68oX9+3b1/cvn0b27dvR0pKCoKDg/H666/j3r17AIBBgwbBw8MDJ06cQEpKCiZNmqS8k1JMTAzy8vJw8OBBnD9/HnPmzIGtrS08PT2RmZkJe3t7zJ8/H5mZmXjzzTfLsLRKb9WqVbCwsMCOHTvwzTffFDuPnZ0dVq5ciYsXL2LBggVYtmwZvvrqqwqJj4gMg7tbdWSou2ZUq1YNK1euxMiRI7F06VIEBwejbdu26N+/Pxo3blzqfhYvXgwHBwesX79eWbzq1aunnD5z5kzExMTg/fffV17f17x5cwBAcnIyjh8/jtu3bytHXPm///s/bNmyBZs2bcKoUaOQkZGhvBcvAPj7+yv7zsjIQHR0NIKCggAAdevWVU5zdXWFRCKBg4NDhe7u9Pf3x5w5cyCTyTQOqfbZZ58p//bx8cGHH36I9evX46OPPqqoMImognFLUkfa3DVD36Kjo3Hz5k388ssv6Ny5M/bv34/g4GCtTnI5c+YMwsLCih0+5vbt27h58ybatm1b7GvPnj2LR48eoWbNmsotW1tbW6Snpyt318bFxWHEiBGIiIjA7NmzVXbjvv/++5gxYwZat26NadOm4dy5c9otgHIQEhLy0nk2bNiA1q1bw9XVFba2tvjss8+QkZFRAdERkaGwSOrI0HfNsLKyQseOHTFlyhQcPnwYQ4YMwbRp0wD8e2cXIf7dyi167Mza2lpj3yVNA4BHjx7Bzc0NZ86cUXmkpaVh4sSJAJ6fUfrHH3+gW7du2LdvHxo0aIDNmzcDAEaMGIG///4bb7/9Ns6fP49mzZph0aJFGt/PzMxMJZfi8ikrGxubEqcfOXIEgwYNQteuXfHbb7/h9OnT+PTTT5Gfn6/XOIjIuLBI6sjY7prRoEEDPH78GADg4uIC4PklDAovnsQDAI0bN0ZSUlKxxcbOzg4+Pj44cOBAse8VHByMW7duoVq1anjllVdUHs7Ozsr56tWrhwkTJmDXrl2IiorCihUrlNM8PT0xevRoJCYm4oMPPsCyZcs05ubi4oKcnBxlfsXlU94OHz4Mb29vfPrpp2jWrBn8/f1x7dq1Co2BiCoei6SODHXXjOzsbHTo0AFr1qzBuXPnkJ6ejo0bN2Lu3Lno2bMngOdbgq+99hpmz56N1NRUHDhwQOV4GgCMHTsWMpkM/fv3x8mTJ3Hp0iWsXr0aaWlpAICpU6di8eLFWLRoES5duoRTp04pt/YiIiIQGhqKXr16YdeuXbh69SoOHz6MTz/9FCdPnsSTJ08wduxY7N+/H9euXcOhQ4dw4sQJBAYGAgBiY2Oxc+dOpKen49SpU/j999+V04rTsmVLVK9eHZ988gmuXLmCtWvX6uX6SW34+/sjIyMD69evx5UrV7Bw4ULlljERVV0skjoy1F0zbG1t0bJlS3z11VcIDw9Ho0aNMGXKFIwcORJff/21cr7vv/8ez549Q0hIiHLw6hfVrFkT+/btw6NHj9C2bVuEhIRg2bJlymOUgwcPxsyZM7FkyRI0bNgQ3bt3x6VLl57nJ5Fg27ZtCA8Px9ChQ1GvXj30798f165dQ+3atSGVSpGdnY133nkH9erVQ79+/dClSxckJCQAAAoLCxETE4PAwEB07twZ9erV03hGKQA4OTlhzZo12LZtG4KCgrBu3TrEx8frdbm+zBtvvIEJEyZg7NixaNq0KQ4fPowpU6ZUaAxEVPEkoujBnipGJpPBwcEBDx8+VDtr8enTp0hPT4evr+9Lhy6Sy+XKMx9fHM2huNHFPe09Mb/z/Ep91wxN+VZl5Z2zNp+3ilJQUIBt27aha9euxZ7EVdWYWr4Ac1bkXFItKAkvASkj3jWDiKjqYpHUA8VdM4iIqGoxjf1oREREOmCRJCIi0oBFElC7UJ2oPPBzRlT5mHSRVJz1lJuba+BIyBQoPmemcoYhUVVg0ifuSKVSODo64vbt2wCA6tWraxx+SS6XIz8/H0+fPjWJSyJMLV+g/HIWQiA3Nxe3b9+Go6MjpFKe+UxUWZh0kQSgHGlCUSg1EULgyZMnsLa2LvU4hpWZqeULlH/Ojo6OHMiZqJIx+SIpkUjg5uaGWrVqlXjT7IKCAhw8eBDh4eEmsbvM1PIFyjdnc3NzbkESVUImXyQVpFJpiT9iUqkUz549g5WVlUkUDVPLFzDNnImoZKZxsImIiEgHLJJEREQasEgSERFpwGOSVCUVyguN6qbzxhYPEZWOQbckCwsLMWXKFPj6+sLa2hp+fn74/PPPVe5MIoTA1KlT4ebmBmtra0RERCjHNSQqTmJqInwW+KD9qvYYmDgQ7Ve1h88CHySmJjIeItKKQYvknDlzsGTJEnz99ddITU3FnDlzMHfuXCxatEg5z9y5c7Fw4UIsXboUx44dg42NDSIjI/H06VMDRk7GKjE1EX1+6qMyvicA3JDdQJ+f+lR4YTK2eIhIOwYtkocPH0bPnj3RrVs3+Pj4oE+fPujUqROOHz8O4PlW5Pz58/HZZ5+hZ8+eaNy4MX744QfcvHkTW7ZsMWToZIQK5YUYv2M8BNTvkapoi90Ri0J5oUnGQ0TaM+gxyVatWuHbb7/FX3/9hXr16uHs2bNITk7GvHnzAADp6em4desWIiIilK9xcHBAy5YtceTIEfTv31+tz7y8POTl5Smfy2QyAM8vFC/pZgEvo3htWfqoTCpjvskZych+lA1rM2uN89x9dBcH0w+ijVcbtWn6zrms8VSEyriey8LU8gWYc9E2bUmEAYcmkMvl+OSTTzB37lxIpVIUFhbiiy++wOTJkwE839Js3bo1bt68CTc3N+Xr+vXrB4lEgg0bNqj1GR8fj4SEBLX2tWvXonr16uWXDBERGa3c3FwMHDgQDx8+hL29falfZ9AtyZ9++gk//vgj1q5di4YNG+LMmTOIjY2Fu7s7Bg8erFOfkydPRlxcnPK5TCaDp6cnOnXqpNWCKaqgoAC7d+9Gx44dTeJuLJUx3+SMZHRb2+2l820duFXjlqQ+cy5rPBWhMq7nsjC1fAHmrMhZsVdRWwYtkhMnTsSkSZOUu02DgoJw7do1zJo1C4MHD1beDDorK0tlSzIrKwtNmzYttk9LS0tYWlqqtZubm+vlA6KvfiqLypRvuG84atrWxA3ZjWKPA0oggYe9B8J9w0u8/EJfOesrnopQmdazPphavgBz1jV3g564k5ubqzYkkVQqhVwuBwD4+vrC1dUVe/fuVU6XyWQ4duwYQkNDKzRWMn5SMykWdF4A4HkBepHi+fzO8yusIBlbPESkPYMWyR49euCLL77A1q1bcfXqVWzevBnz5s1D7969ATwfoSM2NhYzZszAL7/8gvPnz+Odd96Bu7s7evXqZcjQyUhFBUZhU79NqGNfR6Xdw94Dm/ptQlRglEnHQ0TaMeju1kWLFmHKlCkYM2YMbt++DXd3d7z77ruYOnWqcp6PPvoIjx8/xqhRo/DgwQO0adMGO3bsgJWVlQEjJ2MWFRiFnvV7Gs0dbowtHiIqPYMWSTs7O8yfPx/z58/XOI9EIsH06dMxffr0iguMKj2pmRTtfNoZOgwlY4uHiEqHNzgnIiLSgEWSiIhIAxZJIiIiDVgkiYiINOB4kkREGmg7DijHDa16WCSJiIqRmJqI8TvGqwxz5mHvgQWdFxR7fau281PlwN2tRERFaDsOKMcNrbpYJImIXqDtOKAcN7RqY5EkInpBUkaS2hbhiwQErsuuIykjSaf5qXJhkSQiekFmTqZW82k7P1UuLJJERC9ws3N7+UwvzKft/FS5sEgSEb0gzCsMHvYeasObKUgggae9J8K8wnSanyoXFkkiohdoOw4oxw2t2lgkiYiK0HYcUI4bWnXxZgJERMXQdhxQjhtaNbFIEhFpoO04oBw3tOrh7lYiIiINWCSJiIg0YJEkIiLSgEWSiIhIA564Q2oK5YVIzkgGACRnJCPcN9zgZ+gZ2zh9xhZPVcBl+nK6jG9pbN/lyoZFklQoxsTLfpSNdY3XodvabqhpW9OgY+IZ2zh9xhZPVcBl+nK6jm9pTN/lyoi7W0nJGMfEM7aYjC2eqoDL9OU4vqXhsEgSAOMcE8/YYjK2eKoCLtOX4/iWhsUiSQCMc0w8Y4vJ2OKpCipymRY9PldZigTHtzQsFkkCYJxj4hlbTMYWT1VQUcs0MTURPgt80G1tNwBAt7Xd4LPAp1LsduT4lobFIkkAjHNMPGOLydjiqQoqYplW9uNzHN/SsFgkCYBxjolnbDEZWzxVQXkv06pwfI7jWxoWiyQBMM4x8YwtJmOLpyoo72VaFY7PcXxLwzJokfTx8YFEIlF7xMTEAACePn2KmJgY1KxZE7a2toiOjkZWVpYhQ67SjHFMPGOLydjiqQrKc5lWleNzHN/ScAx6M4ETJ06gsPDf3RwXLlxAx44d0bdvXwDAhAkTsHXrVmzcuBEODg4YO3YsoqKicOjQIUOFXOUpxsQ7mH4QsgsybB241eB36aiIcfq0uTMJxw3Uv/JaplXp+Jyu41sa03e5MjJokXRxcVF5Pnv2bPj5+aFt27Z4+PAhli9fjrVr16JDhw4AgBUrViAwMBBHjx7Fa6+9ZoiQTYLUTIo2Xm2w7cI2tPFqYxRfqvIcp0+XO5Nw3ED9K49lqjg+d0N2o9jjkhJI4GHvUWmOz+kyvqWxfZcrG6O5LV1+fj7WrFmDuLg4SCQSpKSkoKCgABEREcp5AgIC4OXlhSNHjmgsknl5ecjLy1M+l8lkAICCggIUFBToHJ/itWXpozIxlXx/TfsVb29+GwIC1mbWAABrM2vce3QPb296G+gN9Kjfw8BRlh9TWM8LOi7A25vfBgBYmVkBeL6OlcfnOs6HvFAOeaHcYDGWJ1NYx0UVl7Ou+UuEEOr/XhnATz/9hIEDByIjIwPu7u5Yu3Ythg4dqlLwAKBFixZo37495syZU2w/8fHxSEhIUGtfu3YtqlevXi6xExGRccvNzcXAgQPx8OFD2Nvbl/p1RrMluXz5cnTp0gXu7u5l6mfy5MmIi4tTPpfJZPD09ESnTp20WjBFFRQUYPfu3ejYsSPMzc3LFGNlYAr5JmckKy8uB55vXXzf6HsMuzAMT+RPlO1bB25FG682hgix3JnCelYolBfi8LXDyEnNgV2gHVp5tzKJ3Y+mtI4VistZsVdRW0ZRJK9du4Y9e/YgMfHfi3pdXV2Rn5+PBw8ewNHRUdmelZUFV1dXjX1ZWlrC0tJSrd3c3FwvHxB99VNZVOV8b+XeUimGCk/kT1Tab+XeqrLLQKEqr2cFc5gjzDcM21K3Icw3rMrnW5QprOOiXsxZ19yNokiuWLECtWrVQrdu//5XHxISAnNzc+zduxfR0dEAgLS0NGRkZCA0NNRQoRoFjrunH1XpzMeKoMtYhuV9RjK/B/rFZarO4EVSLpdjxYoVGDx4MKpV+zccBwcHDB8+HHFxcXBycoK9vT3GjRuH0NBQkz6zlePu6U9VO/OxPOk6lmF5fU75PdA/LtPiGfyOO3v27EFGRgaGDRumNu2rr75C9+7dER0djfDwcLi6uqrskjU1lf0elMaGdyYpHWMby5DfA/3jMtXM4EWyU6dOEEKgXr16atOsrKywePFi3Lt3D48fP0ZiYmKJxyOrsqpwD0pjxDuTlMzYxjLk90D/uExLZvAiSaVTFe5BaayiAqNwdfxVbB24FcDzs1nTx6ebfIEEjG8sQ34P9I/LtGQskpVEVbkHpbFS3JkEAO9M8gJjG8uQ3wP94zItGYtkJcEzMckQjG0sQ34P9I/LtGQskpUEx4gjQzC2sQz5PdA/LtOSsUhWEjwTkwzB2MYy5PdA/7hMS8YiWYnwTEwyBGMby5DfA/3jMtXM4DcTIO1wLEMyBF3HMiyvzym/B/rHZVo8FslKiGMZkiHoMpZheX5O+T3QPy5TdWXa3Zqfn4+0tDQ8e/ZMX/EQEREZDZ2KZG5uLoYPH47q1aujYcOGyMjIAACMGzcOs2fP1muAREREhqJTkZw8eTLOnj2L/fv3w8rKStkeERGBDRs26C04IiIiQ9LpmOSWLVuwYcMGvPbaa5BI/j1luGHDhrhy5YregiMiIjIknYrknTt3UKtWLbX2x48fqxRNIiKiF1W2MSt1KpLNmjXD1q1bMW7cOABQFsbvvvvO5AdEJiKi4lXGMSt1KpIzZ85Ely5dcPHiRTx79gwLFizAxYsXcfjwYRw4cEDfMRIRUSWnGLOy6JBcijErjfWmBTqduNOmTRucPXsWz549Q1BQEHbt2oVatWrhyJEjCAkJ0XeMRERUiVXmMSu13pIsKCjAu+++iylTpmDZsmXlERMREVUh2oxZaWw3M9B6S9Lc3Bw///xzecRCRERVUGUes1Kn3a29evXCli1b9BwKERFVRZV5zEqdTtzx9/fH9OnTcejQIYSEhMDGxkZl+vvvv6+X4IiIqPJTjFl5Q3aj2OOSEkjgYe9hlGNW6lQkly9fDkdHR6SkpCAlJUVlmkQiYZEkIiIlxZiVfX7qAwkkKoXS2Mes1KlIpqen6zsOIiKqwhRjVhZ3neT8zvON8vIPQA9DZQnx/D8C3mmHiIhKUhnHrNR5qKwffvgBQUFBsLa2hrW1NRo3bozVq1frMzYiIqpiFGNWDggagHY+7Yy6QAI6bknOmzcPU6ZMwdixY9G6dWsAQHJyMkaPHo27d+9iwoQJeg2SiIjIEHQqkosWLcKSJUvwzjvvKNveeOMNNGzYEPHx8SySRERUJei0uzUzMxOtWrVSa2/VqhUyM43vYlAiIiJd6FQkX3nlFfz0009q7Rs2bIC/v3+ZgyIiIjIGOu1uTUhIwJtvvomDBw8qj0keOnQIe/fuLbZ4EpF2KtuYe0RVlU5bktHR0Th27BicnZ2xZcsWbNmyBc7Ozjh+/Dh69+6tVV83btzAW2+9hZo1a8La2hpBQUE4efKkcroQAlOnToWbmxusra0RERGBS5cu6RI2UaWQmJoInwU+aL+qPQYmDkT7Ve3hs8AHiamJhg6NyOTofJ1kSEgI1qxZU6Y3v3//Plq3bo327dtj+/btcHFxwaVLl1CjRg3lPHPnzsXChQuxatUq+Pr6YsqUKYiMjMTFixdhZWVVpvcnMjaVdcw9oqpKpyK5bds2SKVSREZGqrTv3LkTcrkcXbp0KVU/c+bMgaenJ1asWKFs8/X1Vf4thMD8+fPx2WefoWfPngCeX59Zu3ZtbNmyBf3799clfCKj9LIx9ySQIHZHLHrW78ldr0QVRKciOWnSJMyePVutXQiBSZMmlbpI/vLLL4iMjETfvn1x4MAB1KlTB2PGjMHIkSMBPL/93a1btxAREaF8jYODA1q2bIkjR44UWyTz8vKQl5enfC6TyQA8HwezoKBAqzxfpHhtWfqoTEwtX8DwOSdnJCP7UTaszaw1znP30V0cTD+INl5t9PKehs65oplavgBzLtqmLYlQ3FdOC9bW1khNTYWPj49K+9WrV9GwYUM8fvy4VP0odpfGxcWhb9++OHHiBMaPH4+lS5di8ODBOHz4MFq3bo2bN2/Cze3fIVT69esHiUSCDRs2qPUZHx+PhIQEtfa1a9eievXqWmRJRERVRW5uLgYOHIiHDx/C3t6+1K/TaUvSwcEBf//9t1qRvHz5stqwWSWRy+Vo1qwZZs6cCQB49dVXceHCBWWR1MXkyZMRFxenfC6TyeDp6YlOnTpptWCKKigowO7du9GxY0eYm5vr3E9lYWr5AobPOTkjGd3WdnvpfFsHbtXrlqQprWdTyxdgzoqcFXsVtaVTkezZsydiY2OxefNm+Pn5AXheID/44AO88cYbpe7Hzc0NDRo0UGkLDAzEzz//DABwdXUFAGRlZalsSWZlZaFp06bF9mlpaQlLS0u1dnNzc718QPTVT2VhavkChss53DccNW1rvnTMvXDfcL0fkzS19Wxq+QLMWdfcdboEZO7cubCxsUFAQAB8fX3h6+uLgIAA1KxZE//3f/9X6n5at26NtLQ0lba//voL3t7eAJ6fxOPq6oq9e/cqp8tkMhw7dgyhoaG6hE5ktBRj7gH/jrGnYOxj7hFVVTrvbj18+DB2796Ns2fPwtraGk2aNEFYmHajSk+YMAGtWrXCzJkz0a9fPxw/fhzffvstvv32WwDPh9+KjY3FjBkz4O/vr7wExN3dHb169dIldCKjVlnH3COqqrQqkkeOHEF2dja6d+8OiUSCTp06ITMzE9OmTUNubi569eqFRYsWFbu7szjNmzfH5s2bMXnyZEyfPh2+vr6YP38+Bg0apJzno48+wuPHjzFq1Cg8ePAAbdq0wY4dO3iNJFVZlXHMPaKqSqsiOX36dLRr1w7du3cHAJw/fx4jR47E4MGDERgYiC+//BLu7u6Ij48vdZ/du3dX9lcciUSC6dOnY/r06dqESlSpKcbcIyLD0uqY5JkzZ/D6668rn69fvx4tWrTAsmXLEBcXh4ULF/LerUREVGVoVSTv37+P2rVrK58fOHBA5cYBzZs3x/Xr1/UXHRERkQFpVSRr166N9PR0AEB+fj5OnTqF1157TTk9JyenSp5iXCgvRHJGMoDn17IVygsNHBEREVUErYpk165dMWnSJCQlJWHy5MmoXr26yhmt586dU143WVUoRmRQXOTdbW03jshARGQitDpx5/PPP0dUVBTatm0LW1tbrFq1ChYWFsrp33//PTp16qT3IA3lxREZXryfpr5HZCjvsQONsX+Ol6h/2i7TontIyuMmBUSVnVZF0tnZGQcPHsTDhw9ha2sLqVT1C7Vx40bY2trqNUBDqagRGRJTE4u9Jm5B5wV6KcDG2H95x2SKtF2mivmzH2VjXeN16La2G2ra1uQ6ICpCpzvuODg4qBVIAHByclLZsqzMkjKSVH5wihIQuC67jqSMJJ3fQ7GlWvR9FFuqZd2la4z9l3dMpkjbZcp1QFR6OhVJU5CZk6nX+Yp62ZYqAMTuiNX5JCFj7L+8YzJF2i5TrgMi7bBIauBm5/bymbSYr6jy3lI1xv4rYuvc1Gi7TLkOiLTDIqlBmFcYPOw91G40rSCBBJ72ngjz0u5+tQrlvaVqjP2Xd0ymSNtlynVApB0WSQ3Ke0SG8t5SNcb+yzsmU6TtMuU6INIOi2QJFCMy1LGvo9LuYe9R5ss/yntL1Rj7L++YTJG2y5TrgEg7LJIvERUYhavjr2LrwK0Ano8Knz4+vcynyZf3lqox9s/xEvVP22XKdUCkHRbJUpCaSdHGqw0AoI1XG739gJTnlqqx9l/eMZkibZcp1wFR6ek06DLpT3mPHWiM/XO8RP3Tdpkq5j+YfhCyCzJsHbiVd9whKgaLpBEo77EDjbF/jpeof9ouU8Uekm0Xtul1DwlRVcLdrURERBqwSBIREWnAIklERKQBiyQREZEGLJJEREQasEgSERFpwCJJRESkAYskERGRBiySREREGrBIEhERacAiSUREpAGLJBERkQYskkRERBqwSBIREWlg0CIZHx8PiUSi8ggICFBOf/r0KWJiYlCzZk3Y2toiOjoaWVlZBoyYiIhMicG3JBs2bIjMzEzlIzk5WTltwoQJ+PXXX7Fx40YcOHAAN2/eRFQUR00nIqKKYfBBl6tVqwZXV1e19ocPH2L58uVYu3YtOnToAABYsWIFAgMDcfToUbz22msVHSoREZkYgxfJS5cuwd3dHVZWVggNDcWsWbPg5eWFlJQUFBQUICIiQjlvQEAAvLy8cOTIEY1FMi8vD3l5ecrnMpkMAFBQUICCggKd41S8tix9VCamli/AnE2BqeULMOeibdqSCCGEXqLSwfbt2/Ho0SPUr18fmZmZSEhIwI0bN3DhwgX8+uuvGDp0qErBA4AWLVqgffv2mDNnTrF9xsfHIyEhQa197dq1qF69ernkQURExi03NxcDBw7Ew4cPYW9vX+rXGbRIFvXgwQN4e3tj3rx5sLa21qlIFrcl6enpibt372q1YIoqKCjA7t270bFjR5ibm+vcT2VhavkCzNkUcja1fAHmrMhZJpPB2dlZ6yJp8N2tL3J0dES9evVw+fJldOzYEfn5+Xjw4AEcHR2V82RlZRV7DFPB0tISlpaWau3m5uZ6+YDoq5/KwtTyBZizKTC1fAHmrGvuBj+79UWPHj3ClStX4ObmhpCQEJibm2Pv3r3K6WlpacjIyEBoaKgBoyQiIlNh0C3JDz/8ED169IC3tzdu3ryJadOmQSqVYsCAAXBwcMDw4cMRFxcHJycn2NvbY9y4cQgNDeWZrUREVCEMWiT/+ecfDBgwANnZ2XBxcUGbNm1w9OhRuLi4AAC++uormJmZITo6Gnl5eYiMjMQ333xjyJCJiMiEGLRIrl+/vsTpVlZWWLx4MRYvXlxBEREREf3LqI5JEhERGRMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkkiIiINWCSJiIg0YJEkIiLSgEWSiIhIAxZJIiIiDVgkiYiINGCRJCIi0sCoBl2uKgrlhUjKSEJmTibc7NwQ5hUGqZnU0GEREVU6hv49ZZHUs8TURIzfMR7/yP5RtnnYe2BB5wWICowyYGRERJWLMfyecnerHiWmJqLPT31UVigA3JDdQJ+f+iAxNdFAkRERVS7G8nvKIqknhfJCjN8xHgJCbZqiLXZHLArlhRUdGhFRpWJMv6csknqSlJGk9h/PiwQErsuuIykjqQKjIiKqfIzp95RFUk8yczL1Oh8Rkakypt9TFkk9cbNz0+t8RESmyph+T1kk9STMKwwe9h6QQFLsdAkk8LT3RJhXWAVHRkRUuRjT7ymLpJ5IzaRY0HkBAKitWMXz+Z3n83pJIqKXMKbfUxZJPYoKjMKmfptQx76OSruHvQc29dvE6ySJiErJWH5PeTMBPYsKjELP+j15xx0iojIyht9TFslyIDWTop1PO0OHQURU6Rn695S7W4mIiDRgkSQiItKARZKIiEgDFkkiIiINWCSJiIg0MJoiOXv2bEgkEsTGxirbnj59ipiYGNSsWRO2traIjo5GVlaW4YIkIiKTYhRF8sSJE/jvf/+Lxo0bq7RPmDABv/76KzZu3IgDBw7g5s2biIriBflERFQxDF4kHz16hEGDBmHZsmWoUaOGsv3hw4dYvnw55s2bhw4dOiAkJAQrVqzA4cOHcfToUQNGTEREpsLgNxOIiYlBt27dEBERgRkzZijbU1JSUFBQgIiICGVbQEAAvLy8cOTIEbz22mvF9peXl4e8vDzlc5lMBgAoKChAQUGBznEqXluWPioTU8sXYM6mwNTyBZhz0TZtGbRIrl+/HqdOncKJEyfUpt26dQsWFhZwdHRUaa9duzZu3bqlsc9Zs2YhISFBrX3Xrl2oXr16mWPevXt3mfuoTEwtX4A5mwJTyxdgzrm5uTr1YbAief36dYwfPx67d++GlZWV3vqdPHky4uLilM9lMhk8PT3RqVMn2Nvb69xvQUEBdu/ejY4dO8Lc3FwfoRo1U8sXYM6mkLOp5QswZ0XOir2K2jJYkUxJScHt27cRHBysbCssLMTBgwfx9ddfY+fOncjPz8eDBw9UtiazsrLg6uqqsV9LS0tYWlqqtZubm+vlA6KvfioLU8sXYM6mwNTyBZizrrkbrEi+/vrrOH/+vErb0KFDERAQgI8//hienp4wNzfH3r17ER0dDQBIS0tDRkYGQkNDDREyERGZGIMVSTs7OzRq1EilzcbGBjVr1lS2Dx8+HHFxcXBycoK9vT3GjRuH0NBQjSftEBER6ZPBz24tyVdffQUzMzNER0cjLy8PkZGR+OabbwwdFhERmQijKpL79+9XeW5lZYXFixdj8eLFhgmIiIhMmsFvJkBERGSsWCSJiIg0YJEkIiLSgEWSiIhIA6M6cYeIiCqPQnkhkjKSkJmTCTc7N4R5hUFqJjV0WHrFIklERFpLTE3E+B3j8Y/sH2Wbh70HFnRegKjAqjOkIXe3EhGRVhJTE9Hnpz4qBRIAbshuoM9PfZCYmmigyPSPRZKIiEqtUF6I8TvGQ0CoTVO0xe6IRaG8sKJDKxcskkREVGpJGUlqW5AvEhC4LruOpIykCoyq/LBIEhFRqWXmZOp1PmPHIklERKXmZuem1/mMHYskERGVWphXGDzsPSCBpNjpEkjgae+JMK+wCo6sfLBIEhFRqUnNpFjQeQEAqBVKxfP5nedXmeslWSSJiEgrUYFR2NRvE+rY11Fp97D3wKZ+m6rUdZK8mQAREWktKjAKPev35B13iIiIiiM1k6KdTztDh1GuuLuViIhIAxZJIiIiDVgkiYiINKjyxySFeH4vQZlMVqZ+CgoKkJubC5lMBnNzc32EZtRMLV+AOZtCzqaWL8CcFTkraoCiJpRWlS+SOTk5AABPT08DR0JERIaWk5MDBweHUs8vEdqW1UpGLpfj5s2bsLOzg0RS/B0iSkMmk8HT0xPXr1+Hvb29HiM0TqaWL8CcTSFnU8sXYM6KnIUQyMnJgbu7O8zMSn+kscpvSZqZmcHDw0Nv/dnb25vMBw0wvXwB5mwKTC1fgDkD0GoLUoEn7hAREWnAIklERKQBi2QpWVpaYtq0abC0tDR0KBXC1PIFmLMpMLV8AeZcVlX+xB0iIiJdcUuSiIhIAxZJIiIiDVgkiYiINGCRJCIi0oBFshQWL14MHx8fWFlZoWXLljh+/LihQyo38fHxkEgkKo+AgABDh6VXBw8eRI8ePeDu7g6JRIItW7aoTBdCYOrUqXBzc4O1tTUiIiJw6dIlwwSrBy/Ld8iQIWrrvHPnzoYJVk9mzZqF5s2bw87ODrVq1UKvXr2QlpamMs/Tp08RExODmjVrwtbWFtHR0cjKyjJQxGVTmnzbtWuntp5Hjx5toIjLbsmSJWjcuLHyhgGhoaHYvn27crq+1i+L5Ets2LABcXFxmDZtGk6dOoUmTZogMjISt2/fNnRo5aZhw4bIzMxUPpKTkw0dkl49fvwYTZo0weLFi4udPnfuXCxcuBBLly7FsWPHYGNjg8jISDx9+rSCI9WPl+ULAJ07d1ZZ5+vWravACPXvwIEDiImJwdGjR7F7924UFBSgU6dOePz4sXKeCRMm4Ndff8XGjRtx4MAB3Lx5E1FRUQaMWnelyRcARo4cqbKe586da6CIy87DwwOzZ89GSkoKTp48iQ4dOqBnz574448/AOhx/QoqUYsWLURMTIzyeWFhoXB3dxezZs0yYFTlZ9q0aaJJkyaGDqPCABCbN29WPpfL5cLV1VV8+eWXyrYHDx4IS0tLsW7dOgNEqF9F8xVCiMGDB4uePXsaJJ6Kcvv2bQFAHDhwQAjxfJ2am5uLjRs3KudJTU0VAMSRI0cMFabeFM1XCCHatm0rxo8fb7igKkCNGjXEd999p9f1yy3JEuTn5yMlJQURERHKNjMzM0RERODIkSMGjKx8Xbp0Ce7u7qhbty4GDRqEjIwMQ4dUYdLT03Hr1i2Vde7g4ICWLVtW6XW+f/9+1KpVC/Xr18d7772H7OxsQ4ekVw8fPgQAODk5AQBSUlJQUFCgsp4DAgLg5eVVJdZz0XwVfvzxRzg7O6NRo0aYPHkycnNzDRGe3hUWFmL9+vV4/PgxQkND9bp+q/wNzsvi7t27KCwsRO3atVXaa9eujT///NNAUZWvli1bYuXKlahfvz4yMzORkJCAsLAwXLhwAXZ2doYOr9zdunULAIpd54ppVU3nzp0RFRUFX19fXLlyBZ988gm6dOmCI0eOQCqVGjq8MpPL5YiNjUXr1q3RqFEjAM/Xs4WFBRwdHVXmrQrrubh8AWDgwIHw9vaGu7s7zp07h48//hhpaWlITEw0YLRlc/78eYSGhuLp06ewtbXF5s2b0aBBA5w5c0Zv65dFklR06dJF+Xfjxo3RsmVLeHt746effsLw4cMNGBmVl/79+yv/DgoKQuPGjeHn54f9+/fj9ddfN2Bk+hETE4MLFy5UuWPrmmjKd9SoUcq/g4KC4Obmhtdffx1XrlyBn59fRYepF/Xr18eZM2fw8OFDbNq0CYMHD8aBAwf0+h7c3VoCZ2dnSKVStTOisrKy4OrqaqCoKpajoyPq1auHy5cvGzqUCqFYr6a8zuvWrQtnZ+cqsc7Hjh2L3377Db///rvKkHmurq7Iz8/HgwcPVOav7OtZU77FadmyJQBU6vVsYWGBV155BSEhIZg1axaaNGmCBQsW6HX9skiWwMLCAiEhIdi7d6+yTS6XY+/evQgNDTVgZBXn0aNHuHLlCtzc3AwdSoXw9fWFq6uryjqXyWQ4duyYyazzf/75B9nZ2ZV6nQshMHbsWGzevBn79u2Dr6+vyvSQkBCYm5urrOe0tDRkZGRUyvX8snyLc+bMGQCo1Ou5KLlcjry8PP2uX/2eW1T1rF+/XlhaWoqVK1eKixcvilGjRglHR0dx69YtQ4dWLj744AOxf/9+kZ6eLg4dOiQiIiKEs7OzuH37tqFD05ucnBxx+vRpcfr0aQFAzJs3T5w+fVpcu3ZNCCHE7NmzhaOjo/jf//4nzp07J3r27Cl8fX3FkydPDBy5bkrKNycnR3z44YfiyJEjIj09XezZs0cEBwcLf39/8fTpU0OHrrP33ntPODg4iP3794vMzEzlIzc3VznP6NGjhZeXl9i3b584efKkCA0NFaGhoQaMWncvy/fy5cti+vTp4uTJkyI9PV3873//E3Xr1hXh4eEGjlx3kyZNEgcOHBDp6eni3LlzYtKkSUIikYhdu3YJIfS3flkkS2HRokXCy8tLWFhYiBYtWoijR48aOqRy8+abbwo3NzdhYWEh6tSpI958801x+fJlQ4elV7///rsAoPYYPHiwEOL5ZSBTpkwRtWvXFpaWluL1118XaWlphg26DErKNzc3V3Tq1Em4uLgIc3Nz4e3tLUaOHFnp/wksLl8AYsWKFcp5njx5IsaMGSNq1KghqlevLnr37i0yMzMNF3QZvCzfjIwMER4eLpycnISlpaV45ZVXxMSJE8XDhw8NG3gZDBs2THh7ewsLCwvh4uIiXn/9dWWBFEJ/65dDZREREWnAY5JEREQasEgSERFpwCJJRESkAYskERGRBiySREREGrBIEhERacAiSUREpAGLJBERkQYskkQVJD4+Hk2bNtXqNRKJBFu2bCmXePTh6tWrkEgkyvuAElU1LJJEOpBIJCU+4uPj1V7z4YcfqtxwWR+GDBkCiUSC2bNnq7Rv2bIFEolEr+9FZIo4niSRDjIzM5V/b9iwAVOnTkVaWpqyzdbWVvm3EAKFhYWwtbVVadcXKysrzJkzB++++y5q1Kih9/4NIT8/HxYWFoYOg4hbkkS6cHV1VT4cHBwgkUiUz//880/Y2dlh+/btCAkJgaWlJZKTk9V2t544cQIdO3aEs7MzHBwc0LZtW5w6dUrrWCIiIuDq6opZs2ZpnKe4Xb3z58+Hj4+P8vmQIUPQq1cvzJw5E7Vr14ajoyOmT5+OZ8+eYeLEiXBycoKHhwdWrFih1v+ff/6JVq1awcrKCo0aNVIb+PbChQvo0qULbG1tUbt2bbz99tu4e/eucnq7du0wduxYxMbGwtnZGZGRkVovB6LywCJJVE4mTZqE2bNnIzU1FY0bN1abnpOTg8GDByM5ORlHjx6Fv78/unbtipycHK3eRyqVYubMmVi0aBH++eefMsW8b98+3Lx5EwcPHsS8efMwbdo0dO/eHTVq1MCxY8cwevRovPvuu2rvM3HiRHzwwQc4ffo0QkND0aNHD2RnZwMAHjx4gA4dOuDVV1/FyZMnsWPHDmRlZaFfv34qfaxatQoWFhY4dOgQli5dWqY8iPSFRZKonEyfPh0dO3aEn58fnJyc1KZ36NABb731FgICAhAYGIhvv/0Wubm5althpdG7d280bdoU06ZNK1PMTk5OWLhwIerXr49hw4ahfv36yM3NxSeffAJ/f39MnjwZFhYWSE5OVnnd2LFjER0djcDAQCxZsgQODg5Yvnw5AODrr7/Gq6++ipkzZyIgIACvvvoqvv/+e/z+++/466+/lH34+/tj7ty5qF+/PurXr1+mPIj0hUWSqJw0a9asxOlZWVkYOXIk/P394eDgAHt7ezx69AgZGRk6vd+cOXOwatUqpKam6vR6AGjYsCHMzP79WahduzaCgoKUz6VSKWrWrInbt2+rvO7F0d6rVauGZs2aKeM4e/Ysfv/9d+UxWVtbWwQEBAAArly5onxdSEiIznETlReeuENUTmxsbEqcPnjwYGRnZ2PBggXw9vaGpaUlQkNDkZ+fr9P7hYeHIzIyEpMnT8aQIUNUppmZmaHo0LEFBQVqfZibm6s8l0gkxbbJ5fJSx/Xo0SP06NEDc+bMUZvm5uam/Ptly4vIEFgkiQzk0KFD+Oabb9C1a1cAwPXr11VOZtHF7Nmz0bRpU7XdlS4uLrh16xaEEMpLQ/R5bePRo0cRHh4OAHj27BlSUlIwduxYAEBwcDB+/vln+Pj4oFo1/uRQ5cLdrUQG4u/vj9WrVyM1NRXHjh3DoEGDYG1tXaY+g4KCMGjQICxcuFClvV27drhz5w7mzp2LK1euYPHixdi+fXuZ3utFixcvxubNm/Hnn38iJiYG9+/fx7BhwwAAMTExuHfvHgYMGIATJ07gypUr2LlzJ4YOHYrCwkK9xUBUHlgkiQxk+fLluH//PoKDg/H222/j/fffR61atcrc7/Tp09V2hwYGBuKbb77B4sWL0aRJExw/fhwffvhhmd9LYfbs2Zg9ezaaNGmC5ORk/PLLL3B2dgYAuLu749ChQygsLESnTp0QFBSE2NhYODo6qhz/JDJGElH0QAUREREB4JYkERGRRiySREREGrBIEhERacAiSUREpAGLJBERkQYskkRERBqwSBIREWnAIklERKQBiyQREZEGLJJEREQasEgSERFp8P8A89fe9Yi0nfMAAAAASUVORK5CYII=",+ "text/plain": [+ "<Figure size 500x300 with 1 Axes>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "import matplotlib.pyplot as plt\n",+ "\n",+ "trial_logs = compiled_program.trial_logs\n",+ "\n",+ "# Extracting trial numbers, scores, and pruning status\n",+ "trial_numbers = list(trial_logs.keys())\n",+ "scores = [trial_logs[trial]['score'] for trial in trial_numbers]\n",+ "pruning_status = [trial_logs[trial]['pruned'] for trial in trial_numbers]\n",+ "\n",+ "# Plot setup\n",+ "plt.figure(figsize=(5, 3))\n",+ "\n",+ "# Plotting each point\n",+ "for trial_number, score, pruned in zip(trial_numbers, scores, pruning_status):\n",+ " if pruned:\n",+ " plt.scatter(trial_number, score, color='grey', label='Pruned Trial' if 'Pruned Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",+ " else:\n",+ " plt.scatter(trial_number, score, color='green', label='Successful Trial' if 'Successful Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",+ "\n",+ "plt.xlabel('Trial Number')\n",+ "plt.ylabel('Score')\n",+ "plt.title('Trial Scores with Pruning Status')\n",+ "plt.grid(True)\n",+ "plt.legend()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "5_Rwxfa1qZH4"+ },+ "source": [+ "We can also __visualize the best prompts__ discovered by MIPRO as our trials progress... (though note that score increases are also due to the selected fewshot examples, which are not shown here for conciseness)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {+ "id": "NnARfPRHqZH4"+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Basline program | Score: 0:\n",+ "Prompt 1 Instruction: context, question -> answer\n",+ "\n",+ "----------------\n",+ "Best program after 0 trials | Score: 56.5:\n",+ "Prompt 1 Instruction: Provide a yes or no answer based on the given context and question. The context and question are used as input to generate the answer.\n",+ "\n",+ "Best program after 5 trials | Score: 56.5:\n",+ "Prompt 1 Instruction: Provide a yes or no answer based on the given context and question. The context and question are used as input to generate the answer.\n",+ "\n",+ "Best program after 10 trials | Score: 69.5:\n",+ "Prompt 1 Instruction: Given the context and question, generate a logical answer based on the reasoning process.\n",+ "\n",+ "Best program after 15 trials | Score: 69.5:\n",+ "Prompt 1 Instruction: Given the context and question, generate a logical answer based on the reasoning process.\n",+ "\n",+ "Best program after 20 trials | Score: 71.0:\n",+ "Prompt 1 Instruction: Determine if it can be logically concluded that a kayak is nearby based on the given context and question, and generate a yes or no answer using the language model.\n",+ "\n",+ "Best program after 25 trials | Score: 71.0:\n",+ "Prompt 1 Instruction: Determine if it can be logically concluded that a kayak is nearby based on the given context and question, and generate a yes or no answer using the language model.\n",+ "\n"+ ]+ }+ ],+ "source": [+ "best_score = 0\n",+ "\n",+ "def get_signature(predictor):\n",+ " if (hasattr(predictor, 'extended_signature')):\n",+ " return predictor.extended_signature\n",+ " elif (hasattr(predictor, 'signature')):\n",+ " return predictor.signature\n",+ "\n",+ "print(f\"Baseline program | Score: {best_score}:\")\n",+ "for i,predictor in enumerate(program.predictors()):\n",+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",+ "print()\n",+ "\n",+ "print(\"----------------\")\n",+ "\n",+ "for trial_num in compiled_program.trial_logs:\n",+ " program_score = compiled_program.trial_logs[trial_num][\"score\"]\n",+ " program_pruned = compiled_program.trial_logs[trial_num][\"pruned\"]\n",+ " if program_score > best_score and not program_pruned and compiled_program.trial_logs[trial_num][\"full_eval\"]:\n",+ " best_score = program_score\n",+ " best_program_so_far = compiled_program.trial_logs[trial_num][\"program\"]\n",+ " if trial_num % 5 == 0:\n",+ " print(f\"Best program after {trial_num} trials | Score: {best_score}:\")\n",+ " for i,predictor in enumerate(best_program_so_far.predictors()):\n",+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",+ " print()"+ ]+ }+ ],+ "metadata": {+ "colab": {+ "provenance": []+ },+ "kernelspec": {+ "display_name": "dspy_test",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.7"+ },+ "widgets": {+ "application/vnd.jupyter.widget-state+json": {+ "037b10adf9724e058c6468f4ca74ed86": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_942897aa22214745adee56d2c54447e9",+ "placeholder": "",+ "style": "IPY_MODEL_0b49910850f3445abe63b6bc7ac18412",+ "value": " 46.2M/46.2M [00:00<00:00, 62.2MB/s]"+ }+ },+ "09b3f2c456af41cba9264dbb9a724027": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "0a16c7f37b9a4bbe9bfe7e737ea62801": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "0a1744c363f640b5b8939f32f6e72922": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "0b49910850f3445abe63b6bc7ac18412": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "12a9763cd97742f4ab80c0494a398ca6": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_9d4f14862b704d9bb53d9e74365d14ac",+ "IPY_MODEL_56989c76e1534cfd8c9b0da93b3b8bf8",+ "IPY_MODEL_f120c447645446e4a04791b97ce9ae93"+ ],+ "layout": "IPY_MODEL_6a5d5ea44c0d4e4384b8956b48c34f14"+ }+ },+ "134fa83980e142bf82d5b97cfc10da69": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "19fe27a0df5a4d39873dd1031904405c": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "1aab1d48c6124526bee6c74892ecd953": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "1bb155669e2a441d8992030e8aaa84a9": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "1c7d71e42c8c494f867b30d781beb681": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "1f23a95e292249a19055f70cda2622a3": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_33267de625db44c2a63d7c7d412a7e61",+ "max": 7405,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_19fe27a0df5a4d39873dd1031904405c",+ "value": 7405+ }+ },+ "20d8df0ee0a4442582686846757bcc7f": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "287c9f993cd44039923fdc122cd9e040": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "29cab51fec0b4dacaa8f829ff217c839": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "2a75d45acfb44463af974acb7a1b0d8e": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "2d47a6a66054438d8e9a3c5e5767c056": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "2f1e652cfa514054abfda794bdfa61a5": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "33267de625db44c2a63d7c7d412a7e61": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "348bb4fff900492cba8333156626a947": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_697eaeb1a004493a9260a3a89671a9ca",+ "IPY_MODEL_8a64950ea896468da3d10f46e9718ec8",+ "IPY_MODEL_7b0b45020d0f45288e2f0e4ceff22524"+ ],+ "layout": "IPY_MODEL_ad21e20d1f1b4b6ea74fdab03af8acdb"+ }+ },+ "35e4e99036894a2ba37d9ba23581a8ff": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "38abb8f460d24c27a66c54bb0417f8ce": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "398d7d36bbd041fe81ec633e973fc504": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_41961130caaf4537a4c1793574c785d7",+ "placeholder": "",+ "style": "IPY_MODEL_e9935b60c48d46469904492e20f9c2e8",+ "value": "Generating validation split: 100%"+ }+ },+ "3b57d0a9768f4befb7509581289035ad": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "41961130caaf4537a4c1793574c785d7": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "43ed7af1d9c84ac8a6ec2195e48e60eb": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "4cd1dc71c80b401da19ed52ef5148d60": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_de6507e9f45741218bf31a9af728b2bf",+ "placeholder": "",+ "style": "IPY_MODEL_38abb8f460d24c27a66c54bb0417f8ce",+ "value": "Generating test split: 100%"+ }+ },+ "52f1714412df4ec78f6ff7d0f8a69862": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_988055b44cf9411e8d45d8a4185fef07",+ "IPY_MODEL_59be08d44c824d76b8525df14191d569",+ "IPY_MODEL_6fde64f36dd84d8eae670efe95e2313a"+ ],+ "layout": "IPY_MODEL_66e6502a463145daa440e967d8bdfdca"+ }+ },+ "531c380db44a404ebb168648ea77c3f4": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_74d58314b1c449e0b7dec3bda8b653c2",+ "placeholder": "",+ "style": "IPY_MODEL_8b7b2b9489ae49049befae848d0fb1a1",+ "value": "Downloading builder script: 100%"+ }+ },+ "567646442eb940b09456328d49248945": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "56989c76e1534cfd8c9b0da93b3b8bf8": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_9a733957f8fc446fbde053ba87289c71",+ "max": 90447,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_287c9f993cd44039923fdc122cd9e040",+ "value": 90447+ }+ },+ "56b80fc45dd247deadceffe17557f0f9": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "598c6f37331e448889b175393a00deff": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "59be08d44c824d76b8525df14191d569": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_a105ef4f1f754c44b7bc0ec8edf0c0cc",+ "max": 47454698,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_1c7d71e42c8c494f867b30d781beb681",+ "value": 47454698+ }+ },+ "5cfebfd6308349038f9cd7ad5bc00fe5": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "6028a5e08f8641f5b8e8182e50f55419": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_09b3f2c456af41cba9264dbb9a724027",+ "placeholder": "",+ "style": "IPY_MODEL_e2f3a3377ad64a4cb9f3a42c1ac97344",+ "value": " 7405/7405 [00:02<00:00, 3033.88 examples/s]"+ }+ },+ "62d8196169bb4a76a94c16d6f1ca61a6": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "66e6502a463145daa440e967d8bdfdca": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "69412c7605ec48859baef6f31c91c520": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_dba7ce7c756d468b94d0bc8f4815ab6f",+ "placeholder": "",+ "style": "IPY_MODEL_926cbb119ea745cf9e344c0e50b75f62",+ "value": " 7405/7405 [00:04<00:00, 1868.04 examples/s]"+ }+ },+ "697eaeb1a004493a9260a3a89671a9ca": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_1aab1d48c6124526bee6c74892ecd953",+ "placeholder": "",+ "style": "IPY_MODEL_598c6f37331e448889b175393a00deff",+ "value": "Downloading data files: 100%"+ }+ },+ "6a5d5ea44c0d4e4384b8956b48c34f14": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "6ba50c3ec9e542d5a8d2f900fbcb8689": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "6c0432c6cd8f4cae9aeb8c2870b39922": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_99b8f3882032404990f2b453111a63ba",+ "IPY_MODEL_6c21df0657ce486381ba2310c7fa0029",+ "IPY_MODEL_a20ac344a32340c785cd162fd0eb55b5"+ ],+ "layout": "IPY_MODEL_6ba50c3ec9e542d5a8d2f900fbcb8689"+ }+ },+ "6c21df0657ce486381ba2310c7fa0029": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_2a75d45acfb44463af974acb7a1b0d8e",+ "max": 566426227,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_ae814b8e55454e0aadc48f7e595575ee",+ "value": 566426227+ }+ },+ "6ecbf23579324112981d4bce0c0ce369": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "6fde64f36dd84d8eae670efe95e2313a": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_2f1e652cfa514054abfda794bdfa61a5",+ "placeholder": "",+ "style": "IPY_MODEL_b17f4731e8e44858889114c52b8f3dd4",+ "value": " 47.5M/47.5M [00:00<00:00, 69.2MB/s]"+ }+ },+ "74d58314b1c449e0b7dec3bda8b653c2": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "782af098d37d4543a4a01ecfed4e5da2": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_62d8196169bb4a76a94c16d6f1ca61a6",+ "max": 46213747,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_0a16c7f37b9a4bbe9bfe7e737ea62801",+ "value": 46213747+ }+ },+ "7b0b45020d0f45288e2f0e4ceff22524": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_96ad4454539145aea6549995344160e3",+ "placeholder": "",+ "style": "IPY_MODEL_acc3573e48f14dffb635f8632c6f6e74",+ "value": " 3/3 [00:24<00:00, 5.68s/it]"+ }+ },+ "7d80528c4b1c48318756230b0174923c": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_398d7d36bbd041fe81ec633e973fc504",+ "IPY_MODEL_d01ca0dd46ee4a4daeeee92214bc07d1",+ "IPY_MODEL_6028a5e08f8641f5b8e8182e50f55419"+ ],+ "layout": "IPY_MODEL_567646442eb940b09456328d49248945"+ }+ },+ "81cbe1842465400cba02a46309d98064": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_134fa83980e142bf82d5b97cfc10da69",+ "placeholder": "",+ "style": "IPY_MODEL_35e4e99036894a2ba37d9ba23581a8ff",+ "value": " 6.42k/6.42k [00:00<00:00, 208kB/s]"+ }+ },+ "835a1e675186490692e336da943d635d": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_95bcef02eb794979b7873b81021e4b40",+ "IPY_MODEL_d84324c4c3dc40fea04cc0df1539b4d8",+ "IPY_MODEL_a6213bcdbcb24b1694204e6baeb763d8"+ ],+ "layout": "IPY_MODEL_bc423d0878154adf940062660a8315ad"+ }+ },+ "84bed63c6597486ca6c39b9c0c4d20bd": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "8a64950ea896468da3d10f46e9718ec8": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_20d8df0ee0a4442582686846757bcc7f",+ "max": 3,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_fa4e50aaf53d4edfb9ca3046cbada2db",+ "value": 3+ }+ },+ "8b7b2b9489ae49049befae848d0fb1a1": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "8e396041a4fd4e6db02410a09b7556c7": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_4cd1dc71c80b401da19ed52ef5148d60",+ "IPY_MODEL_1f23a95e292249a19055f70cda2622a3",+ "IPY_MODEL_69412c7605ec48859baef6f31c91c520"+ ],+ "layout": "IPY_MODEL_ee64a46b61454949ade4177c059bcf61"+ }+ },+ "926cbb119ea745cf9e344c0e50b75f62": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "942897aa22214745adee56d2c54447e9": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "954f3cfcb5d44dddbf1d4db9e99c0d70": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "95bcef02eb794979b7873b81021e4b40": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_0a1744c363f640b5b8939f32f6e72922",+ "placeholder": "",+ "style": "IPY_MODEL_bbd2db64988147f1a4f8856d890bb4c7",+ "value": "Downloading readme: 100%"+ }+ },+ "96ad4454539145aea6549995344160e3": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "988055b44cf9411e8d45d8a4185fef07": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_29cab51fec0b4dacaa8f829ff217c839",+ "placeholder": "",+ "style": "IPY_MODEL_6ecbf23579324112981d4bce0c0ce369",+ "value": "Downloading data: 100%"+ }+ },+ "99b8f3882032404990f2b453111a63ba": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_a298045042014c88939a9fe785fccda8",+ "placeholder": "",+ "style": "IPY_MODEL_84bed63c6597486ca6c39b9c0c4d20bd",+ "value": "Downloading data: 100%"+ }+ },+ "9a733957f8fc446fbde053ba87289c71": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "9d4f14862b704d9bb53d9e74365d14ac": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_e3dd508496f44171b0d91aca0e8b16a5",+ "placeholder": "",+ "style": "IPY_MODEL_dc8077859eec4261a28d708ab06a1008",+ "value": "Generating train split: 100%"+ }+ },+ "9f07fa213de247dabcd3f4213d35a97e": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_d6a78aabcca74314a5b4bb98f350693a",+ "placeholder": "",+ "style": "IPY_MODEL_954f3cfcb5d44dddbf1d4db9e99c0d70",+ "value": "Downloading data: 100%"+ }+ },+ "a105ef4f1f754c44b7bc0ec8edf0c0cc": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "a20ac344a32340c785cd162fd0eb55b5": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_ff4bcec9c18e4f04b1f7898daaffeb1a",+ "placeholder": "",+ "style": "IPY_MODEL_e9c64a790d684c9eaf7f49eea003f43c",+ "value": " 566M/566M [00:22<00:00, 69.9MB/s]"+ }+ },+ "a298045042014c88939a9fe785fccda8": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "a5effa5d67534fb4be2e3320c1fb2b9f": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "a6213bcdbcb24b1694204e6baeb763d8": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_43ed7af1d9c84ac8a6ec2195e48e60eb",+ "placeholder": "",+ "style": "IPY_MODEL_ac6822762e6b46bd862d6f923aeb4437",+ "value": " 9.19k/9.19k [00:00<00:00, 402kB/s]"+ }+ },+ "ac6822762e6b46bd862d6f923aeb4437": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "acc3573e48f14dffb635f8632c6f6e74": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "ad21e20d1f1b4b6ea74fdab03af8acdb": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "ae814b8e55454e0aadc48f7e595575ee": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "b17f4731e8e44858889114c52b8f3dd4": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "ba2b4d7ecdbc4512acf180d8a0d602e8": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "bbd2db64988147f1a4f8856d890bb4c7": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "bc423d0878154adf940062660a8315ad": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "c7d10443100e49d28266aef9f644ed47": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "d01ca0dd46ee4a4daeeee92214bc07d1": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_f048b50740e94bd8805c142eac1673b6",+ "max": 7405,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_a5effa5d67534fb4be2e3320c1fb2b9f",+ "value": 7405+ }+ },+ "d50f5eafe90c4684bbdd96569b8ff247": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_9f07fa213de247dabcd3f4213d35a97e",+ "IPY_MODEL_782af098d37d4543a4a01ecfed4e5da2",+ "IPY_MODEL_037b10adf9724e058c6468f4ca74ed86"+ ],+ "layout": "IPY_MODEL_c7d10443100e49d28266aef9f644ed47"+ }+ },+ "d6a78aabcca74314a5b4bb98f350693a": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "d84324c4c3dc40fea04cc0df1539b4d8": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_ba2b4d7ecdbc4512acf180d8a0d602e8",+ "max": 9193,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_1bb155669e2a441d8992030e8aaa84a9",+ "value": 9193+ }+ },+ "dba7ce7c756d468b94d0bc8f4815ab6f": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "dc8077859eec4261a28d708ab06a1008": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "de6507e9f45741218bf31a9af728b2bf": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "e16044d880174c49af557bd12789493f": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HBoxModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HBoxModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HBoxView",+ "box_style": "",+ "children": [+ "IPY_MODEL_531c380db44a404ebb168648ea77c3f4",+ "IPY_MODEL_e8a46c6ac8a54fdbb44b2c8914552e64",+ "IPY_MODEL_81cbe1842465400cba02a46309d98064"+ ],+ "layout": "IPY_MODEL_f8ada809ecdb417ebc8214d860dd6552"+ }+ },+ "e2f3a3377ad64a4cb9f3a42c1ac97344": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "e3dd508496f44171b0d91aca0e8b16a5": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "e8a46c6ac8a54fdbb44b2c8914552e64": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "FloatProgressModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "FloatProgressModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "ProgressView",+ "bar_style": "success",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_2d47a6a66054438d8e9a3c5e5767c056",+ "max": 6422,+ "min": 0,+ "orientation": "horizontal",+ "style": "IPY_MODEL_3b57d0a9768f4befb7509581289035ad",+ "value": 6422+ }+ },+ "e9935b60c48d46469904492e20f9c2e8": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "e9c64a790d684c9eaf7f49eea003f43c": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "DescriptionStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "DescriptionStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "description_width": ""+ }+ },+ "ee64a46b61454949ade4177c059bcf61": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "f048b50740e94bd8805c142eac1673b6": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "f120c447645446e4a04791b97ce9ae93": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "HTMLModel",+ "state": {+ "_dom_classes": [],+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "HTMLModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/controls",+ "_view_module_version": "1.5.0",+ "_view_name": "HTMLView",+ "description": "",+ "description_tooltip": null,+ "layout": "IPY_MODEL_5cfebfd6308349038f9cd7ad5bc00fe5",+ "placeholder": "",+ "style": "IPY_MODEL_56b80fc45dd247deadceffe17557f0f9",+ "value": " 90447/90447 [00:41<00:00, 2721.55 examples/s]"+ }+ },+ "f8ada809ecdb417ebc8214d860dd6552": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ },+ "fa4e50aaf53d4edfb9ca3046cbada2db": {+ "model_module": "@jupyter-widgets/controls",+ "model_module_version": "1.5.0",+ "model_name": "ProgressStyleModel",+ "state": {+ "_model_module": "@jupyter-widgets/controls",+ "_model_module_version": "1.5.0",+ "_model_name": "ProgressStyleModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "StyleView",+ "bar_color": null,+ "description_width": ""+ }+ },+ "ff4bcec9c18e4f04b1f7898daaffeb1a": {+ "model_module": "@jupyter-widgets/base",+ "model_module_version": "1.2.0",+ "model_name": "LayoutModel",+ "state": {+ "_model_module": "@jupyter-widgets/base",+ "_model_module_version": "1.2.0",+ "_model_name": "LayoutModel",+ "_view_count": null,+ "_view_module": "@jupyter-widgets/base",+ "_view_module_version": "1.2.0",+ "_view_name": "LayoutView",+ "align_content": null,+ "align_items": null,+ "align_self": null,+ "border": null,+ "bottom": null,+ "display": null,+ "flex": null,+ "flex_flow": null,+ "grid_area": null,+ "grid_auto_columns": null,+ "grid_auto_flow": null,+ "grid_auto_rows": null,+ "grid_column": null,+ "grid_gap": null,+ "grid_row": null,+ "grid_template_areas": null,+ "grid_template_columns": null,+ "grid_template_rows": null,+ "height": null,+ "justify_content": null,+ "justify_items": null,+ "left": null,+ "margin": null,+ "max_height": null,+ "max_width": null,+ "min_height": null,+ "min_width": null,+ "object_fit": null,+ "object_position": null,+ "order": null,+ "overflow": null,+ "overflow_x": null,+ "overflow_y": null,+ "padding": null,+ "right": null,+ "top": null,+ "visibility": null,+ "width": null+ }+ }+ }+ }+ },+ "nbformat": 4,+ "nbformat_minor": 0+}diff --git a/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb b/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynbindex a863e0e96..0e45d7602 100644--- a/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_MIPRO.ipynb@@ -32,18 +32,18 @@"#### 2) How much will MIPRO cost me to run?\n","Note that __this notebook__ is free to run, because all LM calls have been cached. However, when using an optimizer on your own program, here is a breakdown of the upper bound of the number of calls to the task model and prompt model respectively:\n","\n",- "- **Task model calls**: MIPRO makes up to __O(TxPxM)__ task model calls, where T is the number of trials, P is the number of prompts in the program, and M is the size of the train set. This is because the model is evaluating the program on the train set each trial. In practice, this should be lower given that MIPRO tunes poor trials early (ie. it may stop a trial after running on the first 100 or so examples if performance is poor).\n",+ "- **Task model calls**: MIPRO makes up to __O(TxPxM)__ task model calls if you run without minibatching, where T is the number of trials, P is the number of prompts in the program, and M is the size of the train set. This is because the model is evaluating the program on the train set each trial. In practice, this should be lower given that MIPRO tunes poor trials early (ie. it may stop a trial after running on the first 100 or so examples if performance is poor). If you run **with minibatching** you can reduce calls even further to __O(TxPxB)__ where **B** is the minibatch size. Note that every few steps (a parameter you set) MIPRO will also run a full eval over all **M** examples.\n","\n",- "- **Prompt model calls**: MIPRO makes up to N*P+10 prompt model calls, where N is the number of instruction / fewshot example set candidates to generate for each prompt, and P is the number of prompts in the program. The extra 10 calls comes from generating a summary of the data in the training set, which we use in the meta prompt to create better instructions.\n",+ "- **Prompt model calls**: MIPRO makes up to N*P+10+(P+1) prompt model calls, where N is the number of instruction / fewshot example set candidates to generate for each prompt, and P is the number of prompts in the program. The extra 10 calls comes from generating a summary of the data in the training set, which we use in the meta prompt to create better instructions. The extra (P+1) comes from program summarization where the proposer LLM will look at the program code and try to describe what each module does and what the whole program does.\n","\n","#### 3) How should I configure the hyperparameters?\n","We have yet to run full hyperparameter sweeps with MIPRO, but based off of initial experimintation, we'd recommend the following:\n","- __Trial num__: Gains can be seen after about 20-30 trials. However, 100-200 trials can help with adding on additional marginal gains.\n",- "- __n__: This hyperparameter controls the number of candidate prompts and fewshot example sets that are generated to optimize over. With more trials and less prompts to optimize, we can set n to be higher, as we have more trials to explore different combinations of prompts. If your program has between 2-3 modules and is the `num_trials=30`, we'd recommend ~`n=10`. If n is higher (say `n=100`), then we can go higher to ~`n=15`. If you have a program with only 1 module and are keeping the program 0-shot (ie. no fewshot examples), then `num_trials` should be set to equal `n`, because each trial can explore a new instruction.\n",+ "- __num candidates__: This hyperparameter controls the number of candidate prompts and fewshot example sets that are generated to optimize over. With more trials and less prompts to optimize, we can set n to be higher, as we have more trials to explore different combinations of prompts. If your program has between 2-3 modules and is the `num_trials=30`, we'd recommend ~`n=10`. If n is higher (say `n=100`), then we can go higher to ~`n=15`. If you have a program with only 1 module and are keeping the program 0-shot (ie. no fewshot examples), then `num_trials` should be set to equal `n`, because each trial can explore a new instruction.\n","- __Training set size__: Between 200 and 500 training examples are recommended. Increasing the training set size can help prevent overfitting, but adds to the expense to run.\n","\n","#### 4) What should I do if I want to reduce the cost?\n",- "You can always update hyperparameters accordingly, such as using a smaller train set, using less trials, or using a program with less modules.\n",+ "You can always update hyperparameters accordingly, such as using a smaller train set, using less trials, or using a program with less modules. You should take advantage of minibatching.\n","Alternatively, one strategy would be to optimize using a cheaper task model (ie. locally hosted Llama-2), as initial experiments have shown that prompts optimized for a smaller model also transfer to working well on a larger model.\n"]},@@ -75,7 +75,16 @@"id": "JpijP_d7qZH2","outputId": "c641a4b1-05f5-45cc-d715-4347c526b576"},- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/opt/miniconda3/envs/opt-prompt/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ }+ ],"source": ["%load_ext autoreload\n","%autoreload 2\n",@@ -142,11 +151,17 @@"# openai.api_base = \"https://api.openai.com/v1\"\n","\n","prompt_model_name = \"gpt-3.5-turbo-1106\"\n",- "task_model_name = \"meta-llama/Llama-2-13b-chat-hf\"\n",+ "task_model_name = \"meta-llama/Meta-Llama-3-8B\"\n","colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n","\n",- "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=150)\n",- "task_model = dspy.HFClientTGI(model=task_model_name, port=[7140, 7141, 7142, 7143], max_tokens=150)\n",+ "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=1000, stop=[\"\\n\\n\", \"\\n---\"])\n",+ "task_model = dspy.HFClientVLLM(\n",+ " model=task_model_name,\n",+ " port=7410,\n",+ " url=[\"http://future-hgx-2:7500\", \"http://future-hgx-2:7501\", \"http://future-hgx-2:7502\", \"http://future-hgx-2:7503\", \"http://future-hgx-2:7504\", \"http://future-hgx-2:7505\", \"http://future-hgx-2:7506\", \"http://future-hgx-2:7507\"],\n",+ " max_tokens=1000,\n",+ " stop=[\"\\n\\n\", \"\\n---\", \"assistant\"],\n",+ ")\n","\n","colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n","\n",@@ -288,7 +303,7 @@"name": "stderr","output_type": "stream","text": [- "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.\n",+ "/opt/miniconda3/envs/opt-prompt/lib/python3.11/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by promote_options='default'.\n"," table = cls._concat_blocks(blocks, axis=0)\n"]}@@ -351,7 +366,7 @@"\n","hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)\n","trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:TRAIN_SIZE]\n",- "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",+ "valset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n","\n","# Set up metrics\n","NUM_THREADS = 10\n",@@ -359,7 +374,7 @@"metric = dspy.evaluate.answer_exact_match\n","\n","kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",- "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"+ "evaluate = Evaluate(devset=valset, metric=metric, **kwargs)"]},{@@ -369,7 +384,7 @@},"source": ["### 2] Baseline Evaluation\n",- "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __21.6%__ on our trainset, and __22.6%__ on our devset."+ "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __35.4%__ on our trainset, and __38.2%__ on our devset."]},{@@ -387,43 +402,14 @@"name": "stderr","output_type": "stream","text": [- "Average Metric: 3 / 7 (42.9): 1%| | 6/500 [00:00<00:43, 11.48it/s]"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 108 / 500 (21.6): 100%|██████████| 500/500 [00:33<00:00, 14.99it/s]\n",- "/lfs/0/kristaoo/dspy/dspy/evaluate/evaluate.py:187: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",- " df = df.applymap(truncate_cell)\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 108 / 500 (21.6%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 113 / 500 (22.6): 100%|██████████| 500/500 [00:33<00:00, 14.78it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 113 / 500 (22.6%)\n"+ "Average Metric: 177 / 500 (35.4): 100%|██████████| 500/500 [00:04<00:00, 114.75it/s]\n",+ "Average Metric: 191 / 500 (38.2): 100%|██████████| 500/500 [00:04<00:00, 112.33it/s]\n"]}],"source": ["baseline_train_score = evaluate(program,devset=trainset)\n",- "baseline_eval_score = evaluate(program, devset=devset)"+ "baseline_eval_score = evaluate(program, devset=valset)"]},{@@ -455,31 +441,23 @@"\n","MIPRO only needs a metric, DSPy module, and training set to see huge gains on your task! You can instantiate a MIPRO Optimizer and compile in just two lines:\n","```python\n",- "teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature)\n",- "compiled_program = teleprompter.compile(program, trainset=trainset, num_trials=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)\n",+ "teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature)\n",+ "compiled_program = teleprompter.compile(program, trainset=trainset, devset=devset, num_trials=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)\n","```"]},{"cell_type": "code",- "execution_count": 5,+ "execution_count": null,"metadata": {"id": "NVfMJ_FpBlSI"},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "/lfs/0/kristaoo/dspy/examples/qa/hotpot/MIPRO_notebook_cache/compiler\n"- ]- }- ],+ "outputs": [],"source": ["import cloudpickle as pickle\n","from dspy.teleprompt import MIPRO\n","\n",- "LOAD_PRECOMPILED_PROGRAM = True\n",+ "LOAD_PRECOMPILED_PROGRAM = False\n","\n","# By default, we will load the precompiled program\n","if LOAD_PRECOMPILED_PROGRAM:\n",@@ -491,13 +469,13 @@"else:\n"," # Define hyperparameters:\n"," N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",- " trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial)\n",+ " batches = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial)\n"," temperature = 1.0 # The temperature configured for generating new instructions\n","\n"," # Compile\n"," eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n"," teleprompter = MIPRO(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=N, init_temperature=temperature, verbose=True)\n",- " compiled_program = teleprompter.compile(program, trainset=trainset, num_trials=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"+ " compiled_program = teleprompter.compile(program, trainset=trainset, valset=valset, num_batches=batches, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"]},{@@ -507,7 +485,7 @@},"source": ["#### 3b] Evaluate optimized program\n",- "Now, we evaluate our program that has been optimized with MIPRO. We see that performance on train and dev have improved by __+20pt__ and __+17.8pt__ respectively!"+ "Now, we evaluate our program that has been optimized with MIPRO. We see that performance on train and dev have improved by __+11pt__ and __+6.8pt__ respectively!"]},{@@ -521,48 +499,14 @@"name": "stderr","output_type": "stream","text": [- " 0%| | 0/500 [00:00<?, ?it/s]"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 208 / 500 (41.6): 100%|██████████| 500/500 [00:32<00:00, 15.37it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 208 / 500 (41.6%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 202 / 500 (40.4): 100%|██████████| 500/500 [00:32<00:00, 15.25it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 202 / 500 (40.4%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"+ "Average Metric: 232 / 500 (46.4): 100%|██████████| 500/500 [00:04<00:00, 113.88it/s]\n",+ "Average Metric: 225 / 500 (45.0): 100%|██████████| 500/500 [00:04<00:00, 124.40it/s]\n"]}],"source": ["bayesian_train_score = evaluate(compiled_program, devset=trainset)\n",- "bayesian_eval_score = evaluate(compiled_program, devset=devset)"+ "bayesian_eval_score = evaluate(compiled_program, devset=valset)"]},{@@ -592,7 +536,7 @@"outputs": [{"data": {- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAE8CAYAAACrYErbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABL40lEQVR4nO3deVyU1f4H8M+A7MOAKJuxqgio4IJaZCCaC26pYGZ6u+JWFpRmWnq7Jtri0q1cMuqWqTfDXC5atxQ1V3BLyTWJlNCxRDFKB0TZ5vz+8DXzcxwGmGGGGZjP+/XipfOs3zPPzHyf5zznnEcihBAgIiIiLTbmDoCIiMhSMUkSERHpwCRJRESkA5MkERGRDkySREREOjBJEhER6cAkSUREpAOTJBERkQ5MkkRERDowSVKt4uLiEBcXZ9C6EokEqampRo3HmunzfkokEqSkpJg2oEa2du1aSCQSXLp0ydyhkBVhkmzmJBJJvf72799vthhv3LiB6dOnIywsDE5OTvDy8kKvXr3w2muvobS01GxxWbrDhw8jNTUVN2/eNOp2L126pPHZsLW1RUBAAEaNGoVTp04ZdV9NSXZ2NgYPHoyHHnoIjo6OCAgIwPDhw5Genq5epqysDKmpqQ36PpnquJJhWpg7ADKtL774QuP1f/7zH+zevVtrenh4eI3r79q1y2SxAcCff/6JHj16QKFQYNKkSQgLC0NxcTHOnDmDtLQ0PP/885BKpSaNoam4c+cOWrT4/6/s4cOHsWDBAiQlJcHd3d3o+3v66acxZMgQVFdXIzc3F2lpadixYweOHj2Krl27Gn1/dXnmmWcwduxYODg4NPq+N2/ejKeeegpdu3bF9OnT0bJlSxQUFODgwYP49NNPMW7cOAD3kuSCBQsAwOAaGFMfV9IPk2Qz97e//U3j9dGjR7F7926t6Q8qKyuDs7Mz7O3tTRkeVq9eDblcjkOHDuHRRx/VmKdQKEy+//vdvn0bLi4ujbY/fTk6Ojbq/rp3767xOenduzeeeOIJpKWl4ZNPPqlxHVO+h7a2trC1tTXJtuuSmpqKjh074ujRo1qfyaKiIrPERI2D1a2EuLg4dO7cGTk5OYiNjYWzszP+8Y9/qOfdf0ZcUVGBN954A1FRUXBzc4OLiwtiYmKwb98+g/adn58PW1tbPPLII1rzZDKZVmI4duwYhgwZgpYtW8LFxQWRkZFYvny5xjJ79+5FTEwMXFxc4O7ujhEjRiA3N1djmdTUVEgkEpw/fx7jxo1Dy5Yt8dhjj6nnr1+/HlFRUXBycoKHhwfGjh2LK1euaGzjwoULSExMhI+PDxwdHeHn54exY8fi1q1bOsu7YsUK2NraalSlvffee5BIJJg5c6Z6WnV1NVxdXfHaa6+pp91/TzI1NRWzZ88GAAQHB6urRh+8X7dt2zZ07twZDg4O6NSpEzIzM3XGVpd+/foBAAoKCgD8/z3CAwcO4IUXXoCXlxf8/PwAAElJSQgKCtLahup9v5/q/mldsdZ0TzIoKAjDhg1DdnY2evXqBUdHR7Rt2xb/+c9/tPZ95swZ9OnTB05OTvDz88Nbb72FNWvW1Os+Z35+Pnr27FnjSZuXlxeAe9XUnp6eAIAFCxaoj4nqmJ05cwZJSUlo27YtHB0d4ePjg0mTJqG4uFjj/dF1XFXV4GvXrtWK4cH71SUlJZgxYwaCgoLg4OAALy8vDBgwAD/++GOt5SRtvJIkAEBxcTEGDx6MsWPH4m9/+xu8vb1rXE6hUOCzzz7D008/jalTp6KkpASrV6/GoEGD8MMPP+hdDRcYGIjq6mp88cUXmDBhQq3L7t69G8OGDYOvry+mT58OHx8f5Obm4ttvv8X06dMBAN9//z0GDx6Mtm3bIjU1FXfu3MHKlSvRu3dv/Pjjj1o/3E8++SRCQkLwzjvvQPXUuLfffhvz5s3DmDFjMGXKFNy4cQMrV65EbGwsTp48CXd3d1RUVGDQoEEoLy/Hiy++CB8fH/z+++/49ttvcfPmTbi5udVYhpiYGCiVSmRnZ2PYsGEAgKysLNjY2CArK0u93MmTJ1FaWorY2Ngat5OQkIBffvkFGzZswAcffIDWrVsDgPpHGrh3Dy0jIwMvvPACXF1dsWLFCiQmJkIul6NVq1a1vtc1yc/PBwCtdV944QV4enrijTfewO3bt/XebkNjvXjxIkaPHo3JkydjwoQJ+Pzzz5GUlISoqCh06tQJAPD777+jb9++kEgkmDt3LlxcXPDZZ5/Vu+o2MDAQe/bswW+//aY+EXiQp6en+hbBqFGjkJCQAACIjIwEcO/z++uvv2LixInw8fHBTz/9hH//+9/46aefcPToUUgkklqP640bN+oVKwBMmzYNW7ZsQUpKCjp27Iji4mJkZ2cjNzcX3bt3r/d2CIAgq5KcnCwePOx9+vQRAMTHH3+stXyfPn1Enz591K+rqqpEeXm5xjJ//fWX8Pb2FpMmTdKYDkDMnz+/1niuXbsmPD09BQARFhYmpk2bJtLT08XNmzc1lquqqhLBwcEiMDBQ/PXXXxrzlEql+v9du3YVXl5eori4WD3t9OnTwsbGRvz9739XT5s/f74AIJ5++mmNbV26dEnY2tqKt99+W2P62bNnRYsWLdTTT548KQCIzZs311q+B1VXVwuZTCZeffVVdeytWrUSTz75pLC1tRUlJSVCCCHef/99YWNjo1HWB9/Pd999VwAQBQUFWvsBIOzt7cXFixc13gcAYuXKlbXGWFBQIACIBQsWiBs3bohr166J/fv3i27dugkA4r///a8QQog1a9YIAOKxxx4TVVVVGtuYMGGCCAwM1Nq26n03JFbV/u4vb2BgoAAgDh48qJ5WVFQkHBwcxCuvvKKe9uKLLwqJRCJOnjypnlZcXCw8PDx0vof3W716tTrOvn37innz5omsrCxRXV2tsdyNGzd0fu7Lysq0pm3YsEErfl3HVXVc1qxZo7WdB/fp5uYmkpOTay0T1Q+rWwkA4ODggIkTJ9a5nK2trbrKSalU4s8//0RVVRV69OhhUFWOt7c3Tp8+jWnTpuGvv/7Cxx9/jHHjxsHLywtvvvmm+uru5MmTKCgowIwZM7QaM6iq7woLC3Hq1CkkJSXBw8NDPT8yMhIDBgzA9u3btfY/bdo0jdcZGRlQKpUYM2YM/vjjD/Wfj48PQkJC1NXKqivFnTt3oqysrN7ltbGxwaOPPoqDBw8CAHJzc1FcXIw5c+ZACIEjR44AuHd12blz5wY13Ojfvz/atWunfh0ZGQmZTIZff/21XuvPnz8fnp6e8PHxQVxcHPLz87FkyRL1FZLK1KlTG3yvsCGxduzYETExMerXnp6eCA0N1Vg3MzMT0dHRGjUdHh4eGD9+fL3imzRpEjIzMxEXF4fs7Gy8+eabiImJQUhICA4fPlyvbTg5Oan/f/fuXfzxxx/q2wzGrgZ1d3fHsWPHcPXqVaNu1xoxSRIA4KGHHqp3I5l169YhMjISjo6OaNWqFTw9PfHdd9/Vei+uNr6+vkhLS0NhYSHy8vKwYsUKdfXd6tWrAfx/VV/nzp11bufy5csAgNDQUK154eHh+OOPP7SqA4ODgzVeX7hwAUIIhISEwNPTU+MvNzdX3UgjODgYM2fOxGeffYbWrVtj0KBBWLVqVb3eg5iYGOTk5ODOnTvIysqCr68vunfvji5duqirXLOzszV++A0REBCgNa1ly5b466+/6rX+s88+i927d2PPnj3IyclBUVERXn31Va3lHnwPGzvW+qx7+fJltG/fXmu5mqbpMmjQIOzcuRM3b97EwYMHkZycjMuXL2PYsGH1arzz559/Yvr06fD29oaTkxM8PT3V752h3x1dli5dinPnzsHf3x+9evVCampqvU+OSBPvSRIAzbPc2qxfvx5JSUkYOXIkZs+eDS8vL9ja2mLRokXqRGYoiUSCDh06oEOHDhg6dChCQkLw5ZdfYsqUKQ3abm0eLLdSqYREIsGOHTtqvDq6vzvKe++9h6SkJHz99dfYtWsXXnrpJSxatAhHjx7Ved8KAB577DFUVlbiyJEjyMrKUifDmJgYZGVl4eeff8aNGzcanCR1Xd2prs7rEhISgv79+9e5XE2fnQcb56hUV1fXOL0hsTa0nPpydnZGTEwMYmJi0Lp1ayxYsAA7duyo8576mDFjcPjwYcyePRtdu3aFVCqFUqlEfHw8lEplnfvV5z0dM2YMYmJisHXrVuzatQvvvvsulixZgoyMDAwePLh+BSUATJKkpy1btqBt27bIyMjQ+NLOnz/fqPtp27YtWrZsicLCQgBQV8WdO3dO5w93YGAgACAvL09r3s8//4zWrVvX2T2hXbt2EEIgODgYHTp0qDPOiIgIRERE4J///CcOHz6M3r174+OPP8Zbb72lc51evXrB3t4eWVlZyMrKUrdmjI2Nxaeffoo9e/aoX9dG14+mJWjZsmWNneFVV/uNLTAwEBcvXtSaXtM0ffTo0QMA1J9TXcfkr7/+wp49e7BgwQK88cYb6ukXLlzQWlbXNlq2bAkAWu+rrvfU19cXL7zwAl544QUUFRWhe/fuePvtt5kk9cTqVtKL6qz9/rP0Y8eOqe+l6evYsWM1toj84YcfUFxcrK467d69O4KDg7Fs2TKtHwlVLL6+vujatSvWrVunscy5c+ewa9cuDBkypM54EhISYGtriwULFmhdiQgh1M31FQoFqqqqNOZHRETAxsYG5eXlte7D0dERPXv2xIYNGyCXyzWuJO/cuYMVK1agXbt28PX1rXU7qoRviSOztGvXDrdu3cKZM2fU0woLC7F161azxDNo0CAcOXJEY8SgP//8E19++WW91leduDxIdZ9b9Tl1dnYGoH1MavreAMCyZcu0tqnruMpkMrRu3Vp9P1vlo48+0nhdXV2tVX3r5eWFNm3a1PnZJG28kiS9DBs2DBkZGRg1ahSGDh2KgoICfPzxx+jYsaNBQ8h98cUX+PLLLzFq1ChERUXB3t4eubm5+Pzzz+Ho6Kjur2ljY4O0tDQMHz4cXbt2xcSJE+Hr64uff/4ZP/30E3bu3AkAePfddzF48GBER0dj8uTJ6i4gbm5u9Rr3tF27dnjrrbcwd+5cXLp0CSNHjoSrqysKCgqwdetWPPvss5g1axb27t2LlJQUPPnkk+jQoQOqqqrwxRdfwNbWFomJiXXuJyYmBosXL4abmxsiIiIA3PshCw0NRV5eHpKSkurcRlRUFADg9ddfx9ixY2FnZ4fhw4dbxIAIY8eOxWuvvYZRo0bhpZdeQllZGdLS0tChQwez9NV79dVXsX79egwYMAAvvviiugtIQEAA/vzzzzqvykeMGIHg4GAMHz4c7dq1w+3bt/H999/jf//7H3r27Inhw4cDuFf13LFjR2zcuBEdOnSAh4cHOnfujM6dOyM2NhZLly5FZWUlHnroIezatUvd5/R+tR3XKVOmYPHixZgyZQp69OiBgwcP4pdfftFYv6SkBH5+fhg9ejS6dOkCqVSK77//HsePH8d7771npHfUipinUS2Zi64uIJ06dapx+Qe7gCiVSvHOO++IwMBA4eDgILp16ya+/fbbGpv8ox5dQM6cOSNmz54tunfvLjw8PESLFi2Er6+vePLJJ8WPP/6otXx2drYYMGCAcHV1FS4uLiIyMlKrS8P3338vevfuLZycnIRMJhPDhw8X58+f11hG1RXhxo0bNcb13//+Vzz22GPCxcVFuLi4iLCwMJGcnCzy8vKEEEL8+uuvYtKkSaJdu3bC0dFReHh4iL59+4rvv/++1vKqfPfddwKAGDx4sMb0KVOmCABi9erVWuvU9H6++eab4qGHHhI2NjYa3QYA1NgFIDAwUEyYMKHW2FRdDd59991al1N1yTh+/HiN83ft2iU6d+4s7O3tRWhoqFi/fr3OLiD1iVVXF5ChQ4dqrfvg51aIe912YmJihIODg/Dz8xOLFi0SK1asEADEtWvXai3rhg0bxNixY0W7du2Ek5OTcHR0FB07dhSvv/66UCgUGssePnxYREVFCXt7e41j9ttvv4lRo0YJd3d34ebmJp588klx9epVvY5rWVmZmDx5snBzcxOurq5izJgxoqioSGMb5eXlYvbs2aJLly7q70mXLl3ERx99VGsZqWYSIUx0d5uIyMLNmDEDn3zyCUpLS8025B1ZNt6TJCKrcOfOHY3XxcXF+OKLL/DYY48xQZJOvCdJRFYhOjoacXFxCA8Px/Xr17F69WooFArMmzfP3KGRBWOSJCKrMGTIEGzZsgX//ve/IZFI0L17d6xevbrOrjZk3XhPkoiISAfekyQiItKBSZKIiEiHZn9PUqlU4urVq3B1dbXoYbyIiMh0hBAoKSlBmzZtYGNT/+vDZp8kr169Cn9/f3OHQUREFuDKlSu1PoDgQc0+Sbq6ugK498bIZDKDt1NZWYldu3Zh4MCBsLOzM1Z4FsvayguwzNZQZmsrL8Ayq8qsUCjg7++vzgn11eyTpKqKVSaTNThJOjs7QyaTWcUHzdrKC7DM1lBmaysvwDI/WGZ9b7ux4Q4REZEOTJJEREQ6NPvqViJqGqqV1ciSZ6GwpBC+rr6ICYiBrQ3HVCXzYpIkIrPLyM3A9Mzp+E3xm3qan8wPy+OXIyE8wYyRkbVjdSsRmVVGbgZGbxqtkSAB4HfF7xi9aTQycjPMFBkRkyQRmVG1shrTM6dDQHsIadW0GZkzUK2sbuzQiAAwSRKRGWXJs7SuIO8nIHBFcQVZ8qxGjIro/zFJEpHZFJYUGnU5ImNjkiQis/F19TXqckTGxiRJRGYTExADP5kfJKh5FBQJJPCX+SMmIKaRIyO6h0mSiMzG1sYWy+OXA4BWolS9Xha/jP0lyWyYJInIrBLCE7BlzBY8JHtIY7qfzA9bxmxhP0kyK4tJkosXL4ZEIsGMGTPU0+7evYvk5GS0atUKUqkUiYmJuH79uvmCJCKTSAhPwKXpl7Bvwj6kJ6Rj34R9KJhewARp4aqV1dh/aT82nN2A/Zf216urjr7rGLIPY7KIEXeOHz+OTz75BJGRkRrTX375ZXz33XfYvHkz3NzckJKSgoSEBBw6dMhMkRKRqdja2CIuKM7cYVA9GTJKkr7rWMJITGa/kiwtLcX48ePx6aefomXLlurpt27dwurVq/H++++jX79+iIqKwpo1a3D48GEcPXrUjBETEVk3Q0ZJ0ncdSxmJyexXksnJyRg6dCj69++Pt956Sz09JycHlZWV6N+/v3paWFgYAgICcOTIETzyyCM1bq+8vBzl5eXq1wqFAsC954tVVlYaHKdq3YZsoymxtvICLLM1sLbyAsYvc7WyGq/tfA2ONo41zpdAgjk752BI2yHqBlf6rmPIPu5XU5kNLb9ECKE9HlQj+eqrr/D222/j+PHjcHR0RFxcHLp27Yply5YhPT0dEydO1Eh4ANCrVy/07dsXS5YsqXGbqampWLBggdb09PR0ODs7m6QcRERk2crKyjBu3DjcunULMpms3uuZ7UryypUrmD59Onbv3g1Hx5rPFgwxd+5czJw5U/1aoVDA398fAwcO1OuNeVBlZSV2796NAQMGWMXTva2tvADLbA1ltrbyAsYv85bzWzD5m8l1Lrf6idUY3XG0QesYso/71VRmVa2ivsyWJHNyclBUVITu3burp1VXV+PgwYP48MMPsXPnTlRUVODmzZtwd3dXL3P9+nX4+Pjo3K6DgwMcHBy0ptvZ2RnlA2Ks7TQV1lZegGW2BtZWXsB4ZfZ188Ud5Z16Lafan77rGLKPmtxfZkPLbraGO48//jjOnj2LU6dOqf969OiB8ePHq/9vZ2eHPXv2qNfJy8uDXC5HdHS0ucImIrJqhoySpO86ljQSk9mSpKurKzp37qzx5+LiglatWqFz585wc3PD5MmTMXPmTOzbtw85OTmYOHEioqOjdTbaISIi0zJklCR917GkkZjM3gWkNh988AGGDRuGxMRExMbGwsfHBxkZfABrc2DuDsJEZDhDRknSdx1LGYnJ7F1A7rd//36N146Ojli1ahVWrVplnoDIJCyhg3Bjq1ZWI0uehcKSQvi6+iImIIbjkVKTlhCegBGhI/T6XOu7jiH7MDaLSpLU/Kk6CD/4JHpVB+HmOFanNZ4UkHUwZJQkfdcx90hMFl3dSs1LtbIa0zOnayVIAOppMzJnNKuqV0sZNYSIDMMkSY0mS56llSzuJyBwRXEFWfKsRozKdBr7pID3eYmMj9Wt1GgKSwqNupyl0+ekoKHVSazSNQ1T30vmvWrLxyRJjcbX1deoy1m6xjopsMb7vI3B1CcePLFpGljdSo3GkjoIN1R9qjYb46SgMat0rak619T3knmvuulgkqRGY0kdhBsiIzcDQcuD0HddX4zLGIe+6/oiaHmQ1g9bY5wUNNZ93vqWuTkw9YmHNTZga8qYJKlRWUoHYUPpcwXQGCcFjVGla21XPaY+8bC2BmxNHZMkNbqE8ARcmn4J+ybsQ3pCOvZN2IeC6QUWnyANuQIw9UmBqat0rfGqx9QnHtbWgK2pY8MdMgtzdxA2hKGtVU05aoiqSvd3xe81JjIJJPCT+RlcpduYLXQthalPPKytAVtTxyRJVE8NuQIw1UmBqkp39KbRkECikSiNUaVrjVc9pj7xMPX2ybhY3UpNgiW0rLTUKwBTVulaaplNydT3kptLAzZrwStJ0lKtrEa2PBsAkC3PRmxwrFm/sJbSn8ySrwBMVaVryWU2JdWJR02fu2Xxyxr8uTP19sl4mCRJgyohFZcWY0PkBgxNH4pW0lZm6+BsSR3lTV21aYz49KnSrc/JkCWX2dSj1Zj6CRSW8IQLqhurW+vhwR+Tuqr6LKFq0BCW1tTfEltWNvUuLCqqfo9D04cCAIamD9XZ79ESy9xY/TZVJx5PRzyNuKA4oycwU2+fGo5XknXQ98rKUqoG9VVXQpJAghmZMzAidESjfZEttWVlU78CuP/q3MnGST29tqtzSyqzJdUuNDdKpRJyuRwlJSVwdXVFQEAAbGys+1qKSbIW+v6YNOUvryUmJEtuWdkUu7AADTsZsoQyW+LJXHORm5uLzMxMKBQK9TSZTIb4+HiEh4ebMTLzsu5ThFroW9VniVWD+rDEhGSNLStNramP9tLU47dUubm52LRpk0aCBACFQoFNmzYhNzfXTJGZH5OkDvp+GZv6l9cSE1JjDoiu733npsoST4b00dTjt0RKpRKZmZm1LpOZmQmlUtlIEVkWJkkd9P0yNvUvryU+oaOx+pPp04ilsZiq8Zclngzpo6nHb4nkcrnWFeSDFAoF5HJ5I0VkWZgkddD3y9jUv7yW2sHZ1C0rLa1FryomU7XctMSTIX009fgtUUlJiVGXa26YJHXQ98vYHL68ltjUXxWXKQZEt8T7yKZO2pZ6MlRfTT1+S+Tq6mrU5ZobJkkd9P0yNpcvryohfTfuOwDAd+O+s4gndJiiP5ml3UdurKRtqSdD9dXU47c0AQEBkMlktS4jk8kQEBDQSBFZFibJWuj7ZTT0y2tpgw/Y2tjisYDHAACPBTxm8YndUJZ2H7kxk7alngzVV1N93JolsrGxQXx8fK3LxMfHW21/SfaTrIOqE/XBgoNQnFPgu3Hf1TqWqb6drpvq4APNgaXdR27spK06Gdp+bnuTPBmyhH6bzUV4eDjGjBnDfpI1YJKsB31/TOr75W3Kgw80B5Y2eLelJW2yLuHh4QgNDeWIOw+w7tKbkSU2GmlOlEolLl26hLNnz+LSpUs19vGytPvIzaHxF1kW1TBzwL2uHnX1dbSxsUFQUBAiIiIQFBRk9QkS4JWk2VjiMHDNhT7Da93/yKLi0mL1dHM8ssiSn7hBTY/qe1BaWorIyEikp6dDKpVaffWpvniaYCaW1mikuTBkeC1LasTClptkDBxmznh4JWkmvP9kfPUdXis0NFSrGsmSGrFY0hM3qOlpyPeAtDFJmomlNRppDvQZXisoKKhxgjIQW26SoZrT98AS8DTCTCyt0UhzwOG1iPg9MDYmSTPi/Sfj4vBaRPweGBurW82M95+MRzW8Vm1VTdY8vBZZB34PjItXkhbAFOOS3s/Shr0zFQ6vRcTvgbHxXWrmTPnYJUukGl7rwQGbZTIZxowZw/5hZBX4PTAeVrc2Y9Y67B2H1yL6/+9BQUEBzp07h3HjxiE4OJjfAz3x3WqmrH3YOw6vRXTve6C698gTRcPwHWumLO1ZiURETRGTZDPFYe+IiBqOSbKZ4rB3REQNxyTZTPGxS0REDcck2Uxx2Dui5qc+z0kl4zJrkkxLS0NkZCRkMhlkMhmio6OxY8cO9fy7d+8iOTkZrVq1glQqRWJiIq5fv27GiJsWDntH1Hzk5uZi+fLlWLduHTIyMrBu3TosX76cj70yMbP2k/Tz88PixYsREhICIQTWrVuHESNG4OTJk+jUqRNefvllfPfdd9i8eTPc3NyQkpKChIQEHDp0yJxhNykc9o6o6VM9H/JBqudDcoAA0zFrkhw+fLjG67fffhtpaWk4evQo/Pz8sHr1aqSnp6Nfv34AgDVr1iA8PBxHjx7FI488Yo6QmyQ+domo6eLzIc3LYkbcqa6uxubNm3H79m1ER0cjJycHlZWV6N+/v3qZsLAwBAQE4MiRIzqTZHl5OcrLy9WvVYP8VlZWorKy0uD4VOs2ZBtNibWVF2CZrUFTLK9cLkdpaWmtCbC0tBQFBQU1DlreFMvcUDWV2dDyS4QQ2kOyNKKzZ88iOjoad+/ehVQqRXp6OoYMGYL09HRMnDhRI+EBQK9evdC3b18sWbKkxu2lpqZiwYIFWtPT09Ph7OxskjIQEZFlKysrw7hx43Dr1i2tMW1rY/YrydDQUJw6dQq3bt3Cli1bMGHCBBw4cMDg7c2dOxczZ85Uv1YoFPD398fAgQP1emMeVFlZid27d2PAgAGws7MzeDtNhbWVF2CZjV1mpVKJ3377DaWlpZBKpfDz8zN7dWBTPMZyuRzp6el1Ljdu3DidV5JNrcwNVVOZa3t0WG3MniTt7e3Rvn17AEBUVBSOHz+O5cuX46mnnkJFRQVu3rwJd3d39fLXr1+Hj4+Pzu05ODjAwcFBa7qdnZ1RPiDG2k5TYW3lBVhmY8jNzUVmZqbGD5NMJkN8fLxFNDBpSsc4ODgYUqm0zudD1jV4eVMqs7HcX2ZDy25xd3mVSiXKy8sRFRUFOzs77NmzRz0vLy8Pcrkc0dHRZoyQiGqjaon54I+6qiUmuyzoh8+HNC+zXknOnTsXgwcPRkBAAEpKSpCeno79+/dj586dcHNzw+TJkzFz5kx4eHhAJpPhxRdfRHR0NFu2ElkotsQ0DdXzIS356ry5MmuSLCoqwt///ncUFhbCzc0NkZGR2LlzJwYMGAAA+OCDD2BjY4PExESUl5dj0KBB+Oijj8wZMhHVQi6X13nvR6FQQC6XIygoqHGCaib4nFTzMGuSXL16da3zHR0dsWrVKqxataqRIiKihigpKTHqcqRJ9ZxUajw8BSEio3F1dTXqckTmxiRJREYTEBBQZ1crmUxWY1cFIkvEJElERsOWmNTc8JNKREalaon54BWlTCbjQNzU5Jh9MAEian7YEpOaCyZJIjIJtsSk5oCndURERDowSRIREenAJElERKQDkyQREZEOTJJEREQ6MEkSERHpwCRJRESkA5MkERGRDkySREREOjBJEhER6cAkSUREpAOTJBERkQ4NSpIVFRXIy8tDVVWVseIhIiKyGAYlybKyMkyePBnOzs7o1KkT5HI5AODFF1/E4sWLjRogERGRuRiUJOfOnYvTp09j//79cHR0VE/v378/Nm7caLTgiIiIzMmg50lu27YNGzduxCOPPAKJRKKe3qlTJ+Tn5xstOGoalEolH65LRM2SQUnyxo0b8PLy0pp++/ZtjaRJzV9ubi4yMzOhUCjU02QyGeLj4xEeHm7GyIiIGs6g0/0ePXrgu+++U79WJcbPPvsM0dHRxomMLF5ubi42bdqkkSABQKFQYNOmTcjNzTVTZERExmHQleQ777yDwYMH4/z586iqqsLy5ctx/vx5HD58GAcOHDB2jGSBlEolMjMza10mMzMToaGhrHoloibLoF+vxx57DKdPn0ZVVRUiIiKwa9cueHl54ciRI4iKijJ2jGSB5HK51hXkgxQKhbrlMxFRU6T3lWRlZSWee+45zJs3D59++qkpYqI6VCurkSXPQmFJIXxdfRETEANbG9tGjaGkpMSoyxERWSK9ryTt7Ozw3//+1xSxUD1k5GYgaHkQ+q7ri3EZ49B3XV8ELQ9CRm5Go8bh6upq1OWIiCyRQdWtI0eOxLZt24wcCtUlIzcDozeNxm+K3zSm/674HaM3jW7URBkQEACZTFbrMjKZDAEBAY0UERGR8RnUcCckJAQLFy7EoUOHEBUVBRcXF435L730klGCo/9XrazG9MzpEBBa8wQEJJBgRuYMjAgd0ShVrzY2NoiPj8emTZt0LhMfH89GO0TUpBmUJFevXg13d3fk5OQgJydHY55EImGSNIEseZbWFeT9BASuKK4gS56FuKC4RokpPDwcY8aMYT9JImq2DEqSBQUFxo6D6lBYUmjU5YwlPDwcoaGhHHGHiJolg5Lk/YS4V/3HkXZMy9fV16jLGZONjQ2CgoIafb9ERKZm8On+f/7zH0RERMDJyQlOTk6IjIzEF198YczY6D4xATHwk/lBgppPRiSQwF/mj5iAmEaOjIio+TIoSb7//vt4/vnnMWTIEGzatAmbNm1CfHw8pk2bhg8++MDYMRIAWxtbLI9fDgBaiVL1eln8skbvL0lE1JwZVN26cuVKpKWl4e9//7t62hNPPIFOnTohNTUVL7/8stECpP+XEJ6ALWO2YHrmdI1GPH4yPyyLX4aE8AQzRkdE1PwYlCQLCwvx6KOPak1/9NFHUVjYuA1HrE1CeAJGhI4w+4g7RETWwKDq1vbt29fYP27jxo0ICQlpcFBUO1sbW8QFxeHpiKcRFxTHBElUD6rnngL3xh5WKpVmjoiaAoOuJBcsWICnnnoKBw8eRO/evQEAhw4dwp49e2rtXE5EZA6q556WlpYiMjIS6enpkEql7M9LdTLoSjIxMRHHjh1D69atsW3bNmzbtg2tW7fGDz/8gFGjRhk7RiIig/G5p9QQBveTjIqKwvr1640ZCxGRUfG5p9RQBn0qtm/fjp07d2pN37lzJ3bs2NHgoIiIjIHPPaWGMihJzpkzB9XV1VrThRCYM2dOg4MiIjIGPveUGsqgJHnhwgV07NhRa3pYWBguXrzY4KCIiIyBzz2lhjIoSbq5ueHXX3/Vmn7x4kWtx2bVZtGiRejZsydcXV3h5eWFkSNHIi8vT2OZu3fvIjk5Ga1atYJUKkViYiKuX79uSNhEZGX43FNqKIOS5IgRIzBjxgzk5+erp128eBGvvPIKnnjiiXpv58CBA0hOTsbRo0exe/duVFZWYuDAgbh9+7Z6mZdffhn/+9//sHnzZhw4cABXr15FQgJHliGiuqmee1obPveUamNQ69alS5ciPj4eYWFh8PPzAwBcuXIFsbGx+Ne//lXv7TzY6mzt2rXw8vJCTk4OYmNjcevWLaxevRrp6eno168fAGDNmjUIDw/H0aNH8cgjjxgSPhFZkfufe1paWqqezueeUn0YlCTd3Nxw+PBh7N69G6dPn4aTkxO6dOmCmJiGPYHi1q1bAAAPDw8AQE5ODiorK9G/f3/1MmFhYQgICMCRI0dqTJLl5eUoLy9Xv1a1bKusrERlZaXBsanWbcg2mhJrKy/AMjdn7du3xwsvvIDLly8jNzcXTz31FAIDA2FjY9Psy24tx/h+NZXZ0PJLhOqBkPVw5MgRFBcXY9iwYepp69atw/z581FWVoaRI0di5cqVcHBw0DsQpVKJJ554Ajdv3kR2djYAID09HRMnTtRIegDQq1cv9O3bF0uWLNHaTmpqKhYsWKA1PT09Hc7OznrHRURETV9ZWRnGjRuHW7du1Xmf+n56XUkuXLgQcXFx6iR59uxZTJ06FRMmTEB4eDjeffddtGnTBqmpqXoFDwDJyck4d+6cOkEaau7cuZg5c6b6tUKhgL+/PwYOHKjXG/OgyspK7N69GwMGDICdnV2DYmwKrK28AMtsDWW2tvICLLOqzHX1l9VFryR56tQpvPnmm+rXX331FXr16oVPP/0UAODv74/58+frnSRTUlLw7bff4uDBg+p7nADg4+ODiooK3Lx5E+7u7urp169fh4+PT43bcnBwqPFK1s7OzigfEGNtp6mwtvICLLM1sLbyAiyzoWXXq0nXX3/9BW9vb/XrAwcOYPDgwerXPXv2xJUrV+q9PSEEUlJSsHXrVuzduxfBwcEa86OiomBnZ4c9e/aop+Xl5UEulyM6Olqf0ImIiPSmV5L09vZGQUEBAKCiogI//vijRuOZkpISvbJ1cnIy1q9fj/T0dLi6uuLatWu4du0a7ty5A+BeA6HJkydj5syZ2LdvH3JycjBx4kRER0ezZSsREZmcXtWtQ4YMwZw5c7BkyRJs27YNzs7OGi1az5w5g3bt2tV7e2lpaQCAuLg4jelr1qxBUlISAOCDDz6AjY0NEhMTUV5ejkGDBuGjjz7SJ2wiIiKD6JUk33zzTSQkJKBPnz6QSqVYt24d7O3t1fM///xzDBw4sN7bq0/DWkdHR6xatQqrVq3SJ1QiIqIG0ytJtm7dGgcPHsStW7cglUpha2urMX/z5s2QSqVGDZCIiMhcDB5MoCaqQQCIiIiaAw5YSEREpAOTJBERkQ5MkkRERDowSRIREenAJElERKQDkyQREZEOTJJEREQ6MEkSERHpwCRJRESkA5MkERGRDkySREREOjBJEhER6cAkSUREpAOTJBERkQ5MkkRERDowSRIREenAJElERKQDkyQREZEOTJJEREQ6MEkSERHpwCRJRESkA5MkERGRDkySREREOjBJEhER6cAkSUREpAOTJBERkQ5MkkRERDowSRIREenAJElERKQDkyQREZEOTJJEREQ6MEkSERHpwCRJRESkA5MkERGRDkySREREOjBJEhER6cAkSUREpAOTJBERkQ5MkkRERDowSRIREenQwtwBWIrq6mpUVlbqnF9ZWYkWLVrg7t27qK6ubsTIzMPayguYtsx2dnawtbU16jaJyPTMmiQPHjyId999Fzk5OSgsLMTWrVsxcuRI9XwhBObPn49PP/0UN2/eRO/evZGWloaQkBCjxSCEwLVr13Dz5s06l/Px8cGVK1cgkUiMtn9LZW3lBUxfZnd3d/j4+FjN+0nUHJg1Sd6+fRtdunTBpEmTkJCQoDV/6dKlWLFiBdatW4fg4GDMmzcPgwYNwvnz5+Ho6GiUGFQJ0svLC87Ozjp/wJRKJUpLSyGVSmFj0/xrqa2tvIDpyiyEQFlZGYqKigAAvr6+Rts2EZmWWZPk4MGDMXjw4BrnCSGwbNky/POf/8SIESMAAP/5z3/g7e2Nbdu2YezYsQ3ef3V1tTpBtmrVqtZllUolKioq4OjoaBVJw9rKC5i2zE5OTgCAoqIieHl5seqVqImw2HuSBQUFuHbtGvr376+e5ubmhocffhhHjhzRmSTLy8tRXl6ufq1QKADcu9/04D3H8vJyCCHg6OgIpVJZazxCCPW/dS3bHFhbeQHTl9nR0RFCCNy5cwcODg5G374hVN+J2u7HNyfWVl6AZX5wmr4sNkleu3YNAODt7a0x3dvbWz2vJosWLcKCBQu0pu/atQvOzs4a01q0aAEfHx/cvn273m9gSUlJvZZrLqytvIDpylxRUYE7d+7gwIEDqKqqMsk+DLV7925zh9CorK28AMtcVlZm0DYsNkkaau7cuZg5c6b6tUKhgL+/PwYOHAiZTKax7N27d3HlyhVIpdI673EKIVBSUgJXV1eraHhhbeUFTF/mu3fvwsnJCbGxsUa7p95QlZWV2L17NwYMGAA7Oztzh2Ny1lZegGVWlVlVq6gvi02SPj4+AIDr169rNHS4fv06unbtqnM9BweHGquy7OzstD4g1dXVkEgksLGxqfMelKr6TbV8c2fO8h46dAjTpk3Dzz//jKFDh2Lbtm11rpOamopt27bh1KlTBu/3wTJLJBKtFtcNiUG1zZo+i+ZmiTGZkrWVF2CZDS27xf7aBwcHw8fHB3v27FFPUygUOHbsGKKjo80YmfnduHEDzz//PAICAuDg4AAfHx8MGjQIhw4dMndoRjFz5kx07doVBQUFWLt2bYO3l5qaColEUutfTQoLC3U2LCMi62DWK8nS0lJcvHhR/bqgoACnTp2Ch4cHAgICMGPGDLz11lsICQlRdwFp06ZNvc/sG0u1shpZ8iwUlhTC19UXMQExsLUxXevFxMREVFRUYN26dWjbti2uX7+OPXv2oLi42GT7bEz5+fmYNm0a/Pz8jLK9WbNmYdq0aerXPXv2xLPPPoupU6fWuLyqhauqNoOIrJdZryRPnDiBbt26oVu3bgDuXUF069YNb7zxBgDg1VdfxYsvvohnn30WPXv2RGlpKTIzMy3mfg4AZORmIGh5EPqu64txGePQd11fBC0PQkZuhkn2d/PmTWRlZWHJkiXo27cvAgMD0atXL8ydOxdPPPEEAODSpUuQSCQa1X43b96ERCLB/v371dN++uknDBs2DDKZDK6uroiJiUF+fr56/vr16xEREQEHBwf4+voiJSVFY3tTpkyBp6cnZDIZ+vXrh9OnT6vnnz59Gn379oWrqytkMhmioqJw4sQJAMDly5cxfPhwtGzZEi4uLujUqRO2b9+ujru4uBiTJk2CRCLB2rVrsXbtWri7u2u8D9u2bav3fUOpVAofHx/1n62tLVxdXdWvx44di5SUFLz88sto166d+upRIpFoVPW+9tpr6NChA5ydndG2bVvMmzfPqloMElkjs15JxsXFqZvd10QikWDhwoVYuHBhI0ZVfxm5GRi9aTQENMvwu+J3jN40GlvGbEFCuPYgCQ0hlUohlUqxbds2PPLIIwZ3Jfj9998RGxuLuLg47N27FzKZDIcOHVK3ukxLS8Ps2bOxaNEiDBkyBLdu3dKozn3yySfh5OSEHTt2wM3NDZ988gkef/xx/PLLL/Dw8MD48ePRrVs3pKWlwdbWFqdOnVLfE0hOTkZFRQUOHjwIFxcXnD9/HlKpFP7+/igsLERoaCgWLlyIp556Cm5ubti4cWPD37g6rFu3DtOmTUNmZiakUmmNy7i6umLt2rVo06YNzp49i6lTp8LV1RWvvvqqyeMjIvOw2IY7lq5aWY3pmdO1EiQACAhIIMGMzBkYETrCqFWvLVq0wNq1azF16lR8/PHH6N69O/r06YOxY8ciMjKy3ttZtWoV3Nzc8NVXX6mTV4cOHdTz33nnHSQnJ+Oll15SN9zp2bMnACA7Oxs//PADioqK1En6X//6F7Zt24YtW7bg2WefhVwux+zZsxEWFgYAGkMJyuVyJCYmIiIiAgDQtm1b9TzVsG1ubm6NWt0ZEhKCJUuWQKFQaLWCVvnnP/+p/n9QUBBmzZqFr776ikmSqBmz2IY7li5LnoXfFL/pnC8gcEVxBVnyLKPvOzExEVevXsU333yD+Ph47N+/H927d9erkcupU6cQExNTY4uvoqIiXL16FX369Klx3dOnT6O0tBStWrVSX9lKpVIUFBSoq2tnzpyJKVOmoH///li8eLFGNe5LL72Et956C71798b8+fNx5swZ/d4AE4iKiqpzmY0bN6J3797w8fGBVCrFP//5T8jl8kaIjojMhUnSQIUlhUZdTl+Ojo4YMGAA5s2bh8OHDyMpKQnz588HAPWV3/1V2Q/eO1MNk1aT2uYB9xpc+fr64tSpUxp/eXl5mD17NoB7LUp/+uknDB06FHv37kXHjh2xdetWAMCUKVPw66+/4plnnsHZs2fRo0cPrFy5Uuf+bGxstKrljX0v0MXFpdb5R44cwfjx4zFkyBB8++23OHnyJF5//XVUVFQYNQ4isixMkgbyda3fINX1Xa6hOnbsiNu3bwMAPD09AdzrwqDyYN+9yMhIZGVl1ZhsXF1dERQUhAMHDtS4r+7du+PatWto0aIF2rdvr/HXunVr9XIdOnTAyy+/jF27diEhIQFr1qxRz/P398e0adOQkZGBV155BZ9++qnOsnl6eqKkpERdvprKY2qHDx9GYGAgXn/9dfTo0QMhISG4fPlyo8ZARI2PSdJAMQEx8JP5QYKaW1hKIIG/zB8xATFG3W9xcTH69euH9evX48yZMygoKMDmzZuxdOlS9UDwTk5OeOSRR7B48WLk5ubiwIEDGvfTACAlJQUKhQJjx47FiRMncOHCBXzxxRfIy8sDALzxxhtYtWoVVq5ciQsXLuDHH39UX+31798f0dHRGDlyJHbt2oVLly7h8OHDeP3113HixAncuXMHKSkp2L9/Py5fvoxDhw7h+PHjCA8PBwDMmDEDO3fuREFBAX788Ufs27dPPa8mDz/8MJydnfGPf/wD+fn5SE9PN0r/SX2EhIRALpfjq6++Qn5+PlasWKG+Miai5otJ0kC2NrZYHr8cALQSper1svhlRu8vKZVK8fDDD+ODDz5AbGwsOnfujHnz5mHq1Kn48MMP1ct9/vnnqKqqQlRUlLq/6f1atWqFvXv3orS0FH369EFUVBQ+/fRT9T3KCRMm4J133kFaWho6deqEYcOG4cKFC/fKJ5Fg+/btiI2NxcSJE9GhQweMHTsWly9fhre3N2xtbVFcXIy///3v6NChA8aMGYPBgwerx9Strq5GcnIywsPDER8fjw4dOuCjjz7SWWYPDw+sX78e27dvR0REBDZs2IDU1FSjvq91eeKJJ/Dyyy8jJSUFXbt2xeHDhzFv3rxGjYGIGp9E1NYHoxlQKBRwc3PDrVu3ahy7taCgAMHBwXX2vVQqleqWj/cP05aRm4HpmdM1GvH4y/yxLH6Z0bt/NCZd5W3OTF1mfT5vjaWyshLbt2/HkCFDrGLIMmsrL8Ay3z92q65cUBt2AWmghPAEjAgd0agj7hARUeNgkjQCWxtbxAXFmTsMIiIyMuuoRyMiIjIAkyQREZEOTJJEREQ6MEkSERHpwCRJRESkA5MkERGRDkySREREOjBJUqMICgrCsmXLjLa9tWvXwt3d3awxEFHzxyTZBCUlJUEikUAikcDe3h7t27fHwoULUVVVZe7QDBIUFKQuT01/SUlJWus89dRT+OWXXxo/WCKyKhxxxwiUSiXkcjlKSkrg6uqKgIAAk493Gh8fjzVr1qC8vBzbt29HcnIy7OzsMHfuXK1lKyoqYG9vb9J4GuL48eOorq4GcO+RVImJicjLy1OPr/jg8y0rKyvh5ORU53MviYgaileSDZSbm4vly5dj3bp1yMjIwLp167B8+XLk5uaadL8ODg7w8fFBYGAgnn/+efTv3x/ffPMNgHtXmiNHjsTbb7+NNm3aIDQ0FMC9p3ds27ZNYzvu7u7qx05dunQJEokEGRkZePzxx9GmTRt069YNR44c0VgnOzsbMTExcHJygr+/P1566SWNZz0WFRVh+PDhcHJyQnBwML788stay+Lp6QkfHx/4+PjAw8MDAODl5QUfHx/cvXsX7u7u2LhxI/r06QNHR0d8+eWXWtWt+fn5GDFiBLy9vSGVStGzZ098//33hry1RERqTJINkJubi02bNkGhUGhMVygU2LRpk8kT5f2cnJxQUVGhfr1nzx7k5eVh9+7d+Pbbb/Xa1uuvv46ZM2fi4MGDCAkJwdNPP62uys3Pz0d8fDwSExNx5swZbNy4EdnZ2UhJSVGvn5SUhCtXrmDfvn3YsmULPvroIxQVFTWofHPmzMH06dORm5uLQYMGac0vLS3FkCFDsGfPHpw8eRLx8fEYPnw45HJ5g/ZLRNaN1a0GUiqVyMzMrHWZzMxMhIaGmrTqVQiBPXv2YOfOnXjxxRfV011cXPDZZ58ZVM06a9YsDB06FAqFAqmpqYiIiMDFixcRFhaGRYsWYfz48ZgxYwaAew8jXrFiBfr06YO0tDTI5XLs2LEDP/zwA3r27AkAWL16da0PVa6PGTNmICFB96PHunTpgi5duqhfv/nmm9i6dSu++eYbjQRORKQPJkkDyeVyrSvIBykUCsjlcgQFBRl9/99++y2kUikqKyuhVCoxbtw4jQcRR0REGHwfMjIyUv1/X19fAPeqUMPCwnD69GmcOXNGowpVCAGlUomCggL88ssvaNGiBaKiotTzw8LC9G6J+qAePXrUOr+0tBSpqan47rvvUFhYiKqqKty5c4dXkkTUIEySBiopKTHqcvrq27cv0tLSYG9vjzZt2qBFC81D6eLiorWORCLBg8/Yrqys1Fru/gezSiQSAPeunIF7yei5557DSy+9pLVeQECAyVqc1lSe+82aNQu7d+/Gv/71L7Rv3x5OTk4YPXq0RhU0EZG+mCQN5OrqatTl9OXi4oL27dvrtY6npycKCwvVry9cuICysjK9ttG9e3ecP39e577DwsJQVVWFnJwcdXVrXl4ebt68qdd+9HXo0CEkJSVh1KhRAO4l80uXLpl0n0TU/LHhjoECAgLUXRR0kclkCAgIaKSI6tavXz98+OGHOHnyJE6cOIFp06ZpXDXWx2uvvYbDhw8jJSUFp06dwoULF/D111+r7/uFhoYiPj4ezz33HI4dO4acnBxMmTLF5N01QkJCkJGRgVOnTuH06dMYN26c+uqXiMhQTJIGsrGxQXx8fK3LxMfHm7y/pD7ee+89+Pv7IyYmBuPGjcOsWbPg7Oys1zYiIyNx4MAB/PLLL4iJiUG3bt3wxhtvoE2bNupl1qxZgzZt2qBPnz5ISEjAs88+Cy8vL2MXR8P777+Pli1b4tFHH8Xw4cMxaNAgdO/e3aT7JHqQUqnEpUuXcPbsWVy6dIknas2ARDx4k6qZUSgUcHNzw61bt7Su/O7evYuCggIEBwfD0dGx1u0olUooFArIZDKNxJebm4vMzEyNRjwymQzx8fENbtFpTrrK25yZusz6fN4aS2VlJbZv344hQ4boXavQFJmyvJb6W2Btxxioucy15YLa8J5kA4WHhyM0NLTRR9whIsuh6jP9IFWf6TFjxjTpk2ZrxiRpBDY2Nibp5kFEls9S+kyTafCIERE1gD59pqnpYZIkImoAc/eZJtNikgS0OtgTmQI/Z82TuftMk2lZdZJUtXrSt0M9kSFUnzNraWFoLZpin2mqP6tuuGNrawt3d3f1EyqcnZ3Vw7A9SKlUoqKiAnfv3rWKm+/WVl7AdGUWQqCsrAxFRUVwd3eHra2t0bZN5qfqM11T61YVS+szTfVn1UkSAHx8fACgzkc5CSFw584dODk56UykzYm1lRcwfZnd3d3VnzdqXsLDwzFmzBiL7CdJDWP1SVIikcDX1xdeXl41DvatUllZiYMHDyI2NtYqqsusrbyAactsZ2fHK8hmjn2mmyerT5Iqtra2tf6I2draoqqqCo6OjlaRNKytvIB1lpmMi32mmx+e4hAREenAJElERKQDkyQREZEOzf6epKoDd13DRtWlsrISZWVlUCgUVnG/ytrKC7DM1lBmaysvwDLf/xQQQP9BPZp9klQNBeXv72/mSIiIyNxKSkrg5uZW7+Wb/fMklUolrl69CldX1wb1fVMoFPD398eVK1f0ehZZU2Vt5QVYZmsos7WVF2CZVWUWQqCkpARt2rTRq1tOs7+StLGxgZ+fn9G2J5PJrOaDBlhfeQGW2RpYW3kBlhmAXleQKmy4Q0REpAOTJBERkQ5MkvXk4OCA+fPnw8HBwdyhNAprKy/AMlsDaysvwDI3VLNvuENERGQoXkkSERHpwCRJRESkA5MkERGRDkySREREOjBJ1sOqVasQFBQER0dHPPzww/jhhx/MHZLJpKamQiKRaPyFhYWZOyyjOnjwIIYPH442bdpAIpFg27ZtGvOFEHjjjTfg6+sLJycn9O/fHxcuXDBPsEZQV3mTkpK0jnl8fLx5gjWSRYsWoWfPnnB1dYWXlxdGjhyJvLw8jWXu3r2L5ORktGrVClKpFImJibh+/bqZIm6Y+pQ3Li5O6zhPmzbNTBE3XFpaGiIjI9UDBkRHR2PHjh3q+cY6vkySddi4cSNmzpyJ+fPn48cff0SXLl0waNAgFBUVmTs0k+nUqRMKCwvVf9nZ2eYOyahu376NLl26YNWqVTXOX7p0KVasWIGPP/4Yx44dg4uLCwYNGoS7d+82cqTGUVd5ASA+Pl7jmG/YsKERIzS+AwcOIDk5GUePHsXu3btRWVmJgQMH4vbt2+plXn75Zfzvf//D5s2bceDAAVy9ehUJCQlmjNpw9SkvAEydOlXjOC9dutRMETecn58fFi9ejJycHJw4cQL9+vXDiBEj8NNPPwEw4vEVVKtevXqJ5ORk9evq6mrRpk0bsWjRIjNGZTrz588XXbp0MXcYjQaA2Lp1q/q1UqkUPj4+4t1331VPu3nzpnBwcBAbNmwwQ4TG9WB5hRBiwoQJYsSIEWaJp7EUFRUJAOLAgQNCiHvH1M7OTmzevFm9TG5urgAgjhw5Yq4wjebB8gohRJ8+fcT06dPNF1QjaNmypfjss8+Menx5JVmLiooK5OTkoH///uppNjY26N+/P44cOWLGyEzrwoULaNOmDdq2bYvx48dDLpebO6RGU1BQgGvXrmkcczc3Nzz88MPN+pjv378fXl5eCA0NxfPPP4/i4mJzh2RUt27dAgB4eHgAAHJyclBZWalxnMPCwhAQENAsjvOD5VX58ssv0bp1a3Tu3Blz585FWVmZOcIzuurqanz11Ve4ffs2oqOjjXp8m/0A5w3xxx9/oLq6Gt7e3hrTvb298fPPP5spKtN6+OGHsXbtWoSGhqKwsBALFixATEwMzp07B1dXV3OHZ3LXrl0DgBqPuWpecxMfH4+EhAQEBwcjPz8f//jHPzB48GAcOXIEtra25g6vwZRKJWbMmIHevXujc+fOAO4dZ3t7e7i7u2ss2xyOc03lBYBx48YhMDAQbdq0wZkzZ/Daa68hLy8PGRkZZoy2Yc6ePYvo6GjcvXsXUqkUW7duRceOHXHq1CmjHV8mSdIwePBg9f8jIyPx8MMPIzAwEJs2bcLkyZPNGBmZytixY9X/j4iIQGRkJNq1a4f9+/fj8ccfN2NkxpGcnIxz5841u3vruugq77PPPqv+f0REBHx9ffH4448jPz8f7dq1a+wwjSI0NBSnTp3CrVu3sGXLFkyYMAEHDhww6j5Y3VqL1q1bw9bWVqtF1PXr1+Hj42OmqBqXu7s7OnTogIsXL5o7lEahOq7WfMzbtm2L1q1bN4tjnpKSgm+//Rb79u3TeGSej48PKioqcPPmTY3lm/px1lXemjz88MMA0KSPs729Pdq3b4+oqCgsWrQIXbp0wfLly416fJkka2Fvb4+oqCjs2bNHPU2pVGLPnj2Ijo42Y2SNp7S0FPn5+fD19TV3KI0iODgYPj4+GsdcoVDg2LFjVnPMf/vtNxQXFzfpYy6EQEpKCrZu3Yq9e/ciODhYY35UVBTs7Ow0jnNeXh7kcnmTPM51lbcmp06dAoAmfZwfpFQqUV5ebtzja9y2Rc3PV199JRwcHMTatWvF+fPnxbPPPivc3d3FtWvXzB2aSbzyyiti//79oqCgQBw6dEj0799ftG7dWhQVFZk7NKMpKSkRJ0+eFCdPnhQAxPvvvy9OnjwpLl++LIQQYvHixcLd3V18/fXX4syZM2LEiBEiODhY3Llzx8yRG6a28paUlIhZs2aJI0eOiIKCAvH999+L7t27i5CQEHH37l1zh26w559/Xri5uYn9+/eLwsJC9V9ZWZl6mWnTpomAgACxd+9eceLECREdHS2io6PNGLXh6irvxYsXxcKFC8WJEydEQUGB+Prrr0Xbtm1FbGysmSM33Jw5c8SBAwdEQUGBOHPmjJgzZ46QSCRi165dQgjjHV8myXpYuXKlCAgIEPb29qJXr17i6NGj5g7JZJ566inh6+sr7O3txUMPPSSeeuopcfHiRXOHZVT79u0TALT+JkyYIIS41w1k3rx5wtvbWzg4OIjHH39c5OXlmTfoBqitvGVlZWLgwIHC09NT2NnZicDAQDF16tQmfxJYU3kBiDVr1qiXuXPnjnjhhRdEy5YthbOzsxg1apQoLCw0X9ANUFd55XK5iI2NFR4eHsLBwUG0b99ezJ49W9y6dcu8gTfApEmTRGBgoLC3txeenp7i8ccfVydIIYx3fPmoLCIiIh14T5KIiEgHJkkiIiIdmCSJiIh0YJIkIiLSgUmSiIhIByZJIiIiHZgkiYiIdGCSJCIi0oFJkqiRpKamomvXrnqtI5FIsG3bNpPEYwyXLl2CRCJRjwNK1NwwSRIZQCKR1PqXmpqqtc6sWbM0Blw2hqSkJEgkEixevFhj+rZt2yCRSIy6LyJrxOdJEhmgsLBQ/f+NGzfijTfeQF5ennqaVCpV/18IgerqakilUo3pxuLo6IglS5bgueeeQ8uWLY2+fXOoqKiAvb29ucMg4pUkkSF8fHzUf25ubpBIJOrXP//8M1xdXbFjxw5ERUXBwcEB2dnZWtWtx48fx4ABA9C6dWu4ubmhT58++PHHH/WOpX///vDx8cGiRYt0LlNTVe+yZcsQFBSkfp2UlISRI0finXfegbe3N9zd3bFw4UJUVVVh9uzZ8PDwgJ+fH9asWaO1/Z9//hmPPvooHB0d0blzZ60H3547dw6DBw+GVCqFt7c3nnnmGfzxxx/q+XFxcUhJScGMGTPQunVrDBo0SO/3gcgUmCSJTGTOnDlYvHgxcnNzERkZqTW/pKQEEyZMQHZ2No4ePYqQkBAMGTIEJSUleu3H1tYW77zzDlauXInffvutQTHv3bsXV69excGDB/H+++9j/vz5GDZsGFq2bIljx45h2rRpeO6557T2M3v2bLzyyis4efIkoqOjMXz4cBQXFwMAbt68iX79+qFbt244ceIEMjMzcf36dYwZM0ZjG+vWrYO9vT0OHTqEjz/+uEHlIDIWJkkiE1m4cCEGDBiAdu3awcPDQ2t+v3798Le//Q1hYWEIDw/Hv//9b5SVlWldhdXHqFGj0LVrV8yfP79BMXt4eGDFihUIDQ3FpEmTEBoairKyMvzjH/9ASEgI5s6dC3t7e2RnZ2usl5KSgsTERISHhyMtLQ1ubm5YvXo1AODDDz9Et27d8M477yAsLAzdunXD559/jn379uGXX35RbyMkJARLly5FaGgoQkNDG1QOImNhkiQykR49etQ6//r165g6dSpCQkLg5uYGmUyG0tJSyOVyg/a3ZMkSrFu3Drm5uQatDwCdOnWCjc3//yx4e3sjIiJC/drW1hatWrVCUVGRxnr3P+29RYsW6NGjhzqO06dPY9++fep7slKpFGFhYQCA/Px89XpRUVEGx01kKmy4Q2QiLi4utc6fMGECiouLsXz5cgQGBsLBwQHR0dGoqKgwaH+xsbEYNGgQ5s6di6SkJI15NjY2ePDRsZWVlVrbsLOz03gtkUhqnKZUKusdV2lpKYYPH44lS5ZozfP19VX/v673i8gcmCSJzOTQoUP46KOPMGTIEADAlStXNBqzGGLx4sXo2rWrVnWlp6cnrl27BiGEumuIMfs2Hj16FLGxsQCAqqoq5OTkICUlBQDQvXt3/Pe//0VQUBBatOBPDjUtrG4lMpOQkBB88cUXyM3NxbFjxzB+/Hg4OTk1aJsREREYP348VqxYoTE9Li4ON27cwNKlS5Gfn49Vq1Zhx44dDdrX/VatWoWtW7fi559/RnJyMv766y9MmjQJAJCcnIw///wTTz/9NI4fP478/Hzs3LkTEydORHV1tdFiIDIFJkkiM1m9ejX++usvdO/eHc888wxeeukleHl5NXi7Cxcu1KoODQ8Px0cffYRVq1ahS5cu+OGHHzBr1qwG70tl8eLFWLx4Mbp06YLs7Gx88803aN26NQCgTZs2OHToEKqrqzFw4EBERERgxowZcHd317j/SWSJJOLBGxVEREQEgFeSREREOjFJEhER6cAkSUREpAOTJBERkQ5MkkRERDowSRIREenAJElERKQDkyQREZEOTJJEREQ6MEkSERHpwCRJRESkw/8BttvSUsJRp3EAAAAASUVORK5CYII=",+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAE8CAYAAACrYErbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJo0lEQVR4nO3deVxU9f4/8NcwsgoDIquyiIqACxq4RAqiqeCWCmqmt9zSrzcs1LK0MsHKrXtTNJebmZqGpoTezNxyYVEwQ01NIiUMUxQlZREFnPn8/vDHXMdhWAdmYF7Px4OHnM85c+b9Pp/jvDnbZyRCCAEiIiJSY6TrAIiIiPQViyQREZEGLJJEREQasEgSERFpwCJJRESkAYskERGRBiySREREGrBIEhERacAiSUREpAGLJFUqODgYwcHBtXqtRCJBVFSUVuMxZDXZnhKJBDNnzqzfgBrY5s2bIZFIcPXqVV2HQgaERbKJk0gk1fo5fvy4zmK8ffs2IiMj4e3tDXNzczg4OKBnz5545513UFRUpLO49N3JkycRFRWFe/fuaXW9V69eVdk3pFIp3NzcMGrUKJw7d06r79WYJCcnY/DgwWjdujXMzMzg5uaG4cOHIzY2VrlMcXExoqKi6vT/qb76lWqnma4DoPq1detWlemvvvoKhw8fVmv38fGp8PWHDh2qt9gA4O+//0b37t1RUFCAKVOmwNvbG3l5eTh//jzWrVuHf/7zn7C0tKzXGBqLBw8eoFmz//2XPXnyJKKjozFp0iTY2Nho/f1eeuklDBkyBHK5HOnp6Vi3bh3279+P1NRUdOvWTevvV5WXX34Z48aNg6mpaYO/965du/Diiy+iW7duiIyMRIsWLZCVlYXExERs2LAB48ePB/C4SEZHRwNArc/A1He/Us2wSDZx//jHP1SmU1NTcfjwYbX2pxUXF8PCwgImJib1GR42btyI7OxsnDhxAs8995zKvIKCgnp//yfdv38fzZs3b7D3qykzM7MGfT8/Pz+V/aR379544YUXsG7dOvznP/+p8DX1uQ2lUimkUmm9rLsqUVFR6NixI1JTU9X2ydzcXJ3ERA2Dp1sJwcHB6Ny5M9LS0hAUFAQLCwu8++67ynlP/kVcWlqKDz74AP7+/rC2tkbz5s0RGBiIY8eO1eq9MzMzIZVK8eyzz6rNk8lkaoXh1KlTGDJkCFq0aIHmzZvD19cXMTExKsscPXoUgYGBaN68OWxsbDBixAikp6erLBMVFQWJRIJLly5h/PjxaNGiBfr06aOcv23bNvj7+8Pc3By2trYYN24crl27prKOy5cvIzw8HE5OTjAzM4OLiwvGjRuH/Px8jfmuWrUKUqlU5VTav//9b0gkEsyZM0fZJpfLYWVlhXfeeUfZ9uQ1yaioKMydOxcA4OHhoTw1+vT1uj179qBz584wNTVFp06dcODAAY2xVaV///4AgKysLAD/u0aYkJCA1157DQ4ODnBxcQEATJo0CW3atFFbR/l2f1L59dOqYq3ommSbNm0wbNgwJCcno2fPnjAzM0Pbtm3x1Vdfqb33+fPn0bdvX5ibm8PFxQUfffQRNm3aVK3rnJmZmejRo0eFf7Q5ODgAeHya2t7eHgAQHR2t7JPyPjt//jwmTZqEtm3bwszMDE5OTpgyZQry8vJUto+mfi0/Db5582a1GJ6+Xl1YWIhZs2ahTZs2MDU1hYODAwYOHIgzZ85Umiep45EkAQDy8vIwePBgjBs3Dv/4xz/g6OhY4XIFBQX44osv8NJLL2HatGkoLCzExo0bERISgp9++qnGp+Hc3d0hl8uxdetWTJw4sdJlDx8+jGHDhsHZ2RmRkZFwcnJCeno6vv/+e0RGRgIAfvzxRwwePBht27ZFVFQUHjx4gNWrV6N37944c+aM2gf3mDFj4OnpicWLF6P8W+M+/vhjLFiwAGPHjsWrr76K27dvY/Xq1QgKCsLZs2dhY2OD0tJShISEoKSkBK+//jqcnJxw/fp1fP/997h37x6sra0rzCEwMBAKhQLJyckYNmwYACApKQlGRkZISkpSLnf27FkUFRUhKCiowvWEhYXh999/x/bt27FixQrY2dkBgPJDGnh8DS0+Ph6vvfYarKyssGrVKoSHhyM7OxstW7asdFtXJDMzEwDUXvvaa6/B3t4eH3zwAe7fv1/j9dY11itXrmD06NGYOnUqJk6ciC+//BKTJk2Cv78/OnXqBAC4fv06+vXrB4lEgvnz56N58+b44osvqn3q1t3dHUeOHMFff/2l/EPgafb29spLBKNGjUJYWBgAwNfXF8Dj/fePP/7A5MmT4eTkhF9//RWff/45fv31V6SmpkIikVTar7dv365WrAAwY8YMxMXFYebMmejYsSPy8vKQnJyM9PR0+Pn5VXs9BECQQYmIiBBPd3vfvn0FALF+/Xq15fv27Sv69u2rnH706JEoKSlRWebu3bvC0dFRTJkyRaUdgFi4cGGl8dy8eVPY29sLAMLb21vMmDFDxMbGinv37qks9+jRI+Hh4SHc3d3F3bt3VeYpFArl7926dRMODg4iLy9P2fbLL78IIyMj8corryjbFi5cKACIl156SWVdV69eFVKpVHz88ccq7RcuXBDNmjVTtp89e1YAELt27ao0v6fJ5XIhk8nE22+/rYy9ZcuWYsyYMUIqlYrCwkIhhBCffvqpMDIyUsn16e35ySefCAAiKytL7X0ACBMTE3HlyhWV7QBArF69utIYs7KyBAARHR0tbt++LW7evCmOHz8unnnmGQFAfPvtt0IIITZt2iQAiD59+ohHjx6prGPixInC3d1dbd3l2702sZa/35P5uru7CwAiMTFR2ZabmytMTU3Fm2++qWx7/fXXhUQiEWfPnlW25eXlCVtbW43b8EkbN25UxtmvXz+xYMECkZSUJORyucpyt2/f1rjfFxcXq7Vt375dLX5N/VreL5s2bVJbz9PvaW1tLSIiIirNiaqHp1sJAGBqaorJkydXuZxUKlWeclIoFPj777/x6NEjdO/evVanchwdHfHLL79gxowZuHv3LtavX4/x48fDwcEBH374ofLo7uzZs8jKysKsWbPUbmYoP32Xk5ODc+fOYdKkSbC1tVXO9/X1xcCBA/HDDz+ovf+MGTNUpuPj46FQKDB27FjcuXNH+ePk5ARPT0/laeXyI8WDBw+iuLi42vkaGRnhueeeQ2JiIgAgPT0deXl5mDdvHoQQSElJAfD46LJz5851unFjwIABaNeunXLa19cXMpkMf/zxR7Vev3DhQtjb28PJyQnBwcHIzMzEsmXLlEdI5aZNm1bna4V1ibVjx44IDAxUTtvb28PLy0vltQcOHEBAQIDKmQ5bW1tMmDChWvFNmTIFBw4cQHBwMJKTk/Hhhx8iMDAQnp6eOHnyZLXWYW5urvz94cOHuHPnjvIyg7ZPg9rY2ODUqVO4ceOGVtdriFgkCQDQunXrat8ks2XLFvj6+sLMzAwtW7aEvb099u3bV+m1uMo4Oztj3bp1yMnJQUZGBlatWqU8fbdx40YA/zvV17lzZ43r+fPPPwEAXl5eavN8fHxw584dtdOBHh4eKtOXL1+GEAKenp6wt7dX+UlPT1fepOHh4YE5c+bgiy++gJ2dHUJCQrBmzZpqbYPAwECkpaXhwYMHSEpKgrOzM/z8/NC1a1flKdfk5GSVD/7acHNzU2tr0aIF7t69W63XT58+HYcPH8aRI0eQlpaG3NxcvP3222rLPb0NGzrW6rz2zz//RPv27dWWq6hNk5CQEBw8eBD37t1DYmIiIiIi8Oeff2LYsGHVunnn77//RmRkJBwdHWFubg57e3vltqvt/x1Nli9fjosXL8LV1RU9e/ZEVFRUtf84IlW8JkkAVP/Krcy2bdswadIkjBw5EnPnzoWDgwOkUimWLFmiLGS1JZFI0KFDB3To0AFDhw6Fp6cnvv76a7z66qt1Wm9lns5boVBAIpFg//79FR4dPfk4yr///W9MmjQJ//3vf3Ho0CG88cYbWLJkCVJTUzVetwKAPn36oKysDCkpKUhKSlIWw8DAQCQlJeG3337D7du361wkNR3dlR+dV8XT0xMDBgyocrmK9p2nb84pJ5fLK2yvS6x1zbOmLCwsEBgYiMDAQNjZ2SE6Ohr79++v8pr62LFjcfLkScydOxfdunWDpaUlFAoFQkNDoVAoqnzfmmzTsWPHIjAwELt378ahQ4fwySefYNmyZYiPj8fgwYOrlygBYJGkGoqLi0Pbtm0RHx+v8p924cKFWn2ftm3bokWLFsjJyQEA5am4ixcvavzgdnd3BwBkZGSozfvtt99gZ2dX5eMJ7dq1gxACHh4e6NChQ5VxdunSBV26dMH777+PkydPonfv3li/fj0++ugjja/p2bMnTExMkJSUhKSkJOXdjEFBQdiwYQOOHDminK6Mpg9NfdCiRYsKH4YvP9pvaO7u7rhy5Ypae0VtNdG9e3cAUO6nmvrk7t27OHLkCKKjo/HBBx8o2y9fvqy2rKZ1tGjRAgDUtqumbers7IzXXnsNr732GnJzc+Hn54ePP/6YRbKGeLqVaqT8r/Yn/0o/deqU8lpaTZ06darCOyJ/+ukn5OXlKU+d+vn5wcPDAytXrlT7kCiPxdnZGd26dcOWLVtUlrl48SIOHTqEIUOGVBlPWFgYpFIpoqOj1Y5EhBDK2/ULCgrw6NEjlfldunSBkZERSkpKKn0PMzMz9OjRA9u3b0d2drbKkeSDBw+watUqtGvXDs7OzpWup7zg6+PILO3atUN+fj7Onz+vbMvJycHu3bt1Ek9ISAhSUlJURgz6+++/8fXXX1fr9eV/uDyt/Dp3+X5qYWEBQL1PKvp/AwArV65UW6emfpXJZLCzs1Nezy63du1alWm5XK52+tbBwQGtWrWqct8kdTySpBoZNmwY4uPjMWrUKAwdOhRZWVlYv349OnbsWKsh5LZu3Yqvv/4ao0aNgr+/P0xMTJCeno4vv/wSZmZmyuc1jYyMsG7dOgwfPhzdunXD5MmT4ezsjN9++w2//vorDh48CAD45JNPMHjwYAQEBGDq1KnKR0Csra2rNe5pu3bt8NFHH2H+/Pm4evUqRo4cCSsrK2RlZWH37t2YPn063nrrLRw9ehQzZ87EmDFj0KFDBzx69Ahbt26FVCpFeHh4le8TGBiIpUuXwtraGl26dAHw+IPMy8sLGRkZmDRpUpXr8Pf3BwC89957GDduHIyNjTF8+HC9GBBh3LhxeOeddzBq1Ci88cYbKC4uxrp169ChQwedPKv39ttvY9u2bRg4cCBef/115SMgbm5u+Pvvv6s8Kh8xYgQ8PDwwfPhwtGvXDvfv38ePP/6IvXv3okePHhg+fDiAx6eeO3bsiG+++QYdOnSAra0tOnfujM6dOyMoKAjLly9HWVkZWrdujUOHDimfOX1SZf366quvYunSpXj11VfRvXt3JCYm4vfff1d5fWFhIVxcXDB69Gh07doVlpaW+PHHH3H69Gn8+9//1tIWNSC6uamWdEXTIyCdOnWqcPmnHwFRKBRi8eLFwt3dXZiamopnnnlGfP/99xXe8o9qPAJy/vx5MXfuXOHn5ydsbW1Fs2bNhLOzsxgzZow4c+aM2vLJycli4MCBwsrKSjRv3lz4+vqqPdLw448/it69ewtzc3Mhk8nE8OHDxaVLl1SWKX8U4fbt2xXG9e2334o+ffqI5s2bi+bNmwtvb28REREhMjIyhBBC/PHHH2LKlCmiXbt2wszMTNja2op+/fqJH3/8sdJ8y+3bt08AEIMHD1Zpf/XVVwUAsXHjRrXXVLQ9P/zwQ9G6dWthZGSk8tgAgAofAXB3dxcTJ06sNLbyRw0++eSTSpcrfyTj9OnTFc4/dOiQ6Ny5szAxMRFeXl5i27ZtGh8BqU6smh4BGTp0qNprn95vhXj82E5gYKAwNTUVLi4uYsmSJWLVqlUCgLh582aluW7fvl2MGzdOtGvXTpibmwszMzPRsWNH8d5774mCggKVZU+ePCn8/f2FiYmJSp/99ddfYtSoUcLGxkZYW1uLMWPGiBs3btSoX4uLi8XUqVOFtbW1sLKyEmPHjhW5ubkq6ygpKRFz584VXbt2Vf4/6dq1q1i7dm2lOVLFJELU09VtIiI9N2vWLPznP/9BUVGRzoa8I/3Ga5JEZBAePHigMp2Xl4etW7eiT58+LJCkEa9JEpFBCAgIQHBwMHx8fHDr1i1s3LgRBQUFWLBgga5DIz3GIklEBmHIkCGIi4vD559/DolEAj8/P2zcuLHKR23IsPGaJBERkQa8JklERKQBiyQREZEGTf6apEKhwI0bN2BlZaXXw3gREVH9EUKgsLAQrVq1gpFR9Y8Pm3yRvHHjBlxdXXUdBhER6YFr165V+gUET2vyRdLKygrA4w0jk8lqvZ6ysjIcOnQIgwYNgrGxsbbC01uGli/AnA0hZ0PLF2DO5TkXFBTA1dVVWROqq8kXyfJTrDKZrM5F0sLCAjKZzCB2NEPLF2DOhpCzoeULMOenc67pZTfeuENERKQBiyQREZEGLJJERFoiV8hx/OpxbL+wHcevHodcIdd5PMnZyQCA5OxkncfTGOn8muT169fxzjvvYP/+/SguLkb79u2xadMm5Td+CyGwcOFCbNiwAffu3UPv3r2xbt06eHp6ai0GIQQePXoEuVzzDlRWVoZmzZrh4cOHlS7XVBhavkD95iyVStGsWTM+htSExafHI/JAJP4q+EvZ5iJzQUxoDMJ8wnQWT15RHrb7bsfQ2KFoadlSZ/E0Vjotknfv3kXv3r3Rr18/7N+/H/b29rh8+TJatGihXGb58uVYtWoVtmzZAg8PDyxYsAAhISG4dOkSzMzM6hxDaWkpcnJyUFxcXOlyQgg4OTnh2rVrBvFBZ2j5AvWfs4WFBZydnWFiYqL1dZNuxafHY/TO0RBQHeXzesF1jN45GnFj4xq0MD0Zj7mRuc7jacx0WiSXLVsGV1dXbNq0Sdnm4eGh/F0IgZUrV+L999/HiBEjAABfffUVHB0dsWfPHowbN65O769QKJCVlQWpVIpWrVrBxMRE44ejQqFAUVERLC0ta/QgamNlaPkC9ZezEAKlpaW4ffs2srKy4OnpaTDb1BDIFXJEHohUK5AAICAggQSzDszCCK8RkBrV/1dy6Vs8jZ1Oi+R3332HkJAQjBkzBgkJCWjdujVee+01TJs2DQCQlZWFmzdvYsCAAcrXWFtbo1evXkhJSamwSJaUlKCkpEQ5XVBQAODxqbSysjK1ZeVyOVq3bg0LC4tKYy3/oDM1NTWIIytDyxeo35xNTU0hlUqRnZ2N4uJimJqaanX9tVX+f+Lp/xtNVX3km5ydjLyiPJUjtqfdKbqDxKxE9HHro7X3rW48T//b0PHoQkX9XNs+1+m3gJSfLp0zZw7GjBmD06dPIzIyEuvXr8fEiRNx8uRJ9O7dGzdu3ICzs7PydWPHjoVEIsE333yjts6oqChER0ertcfGxqoVwmbNmsHJyQmurq48BUb1rrS0FNeuXcPNmzfx6NEjXYdDZFCKi4sxfvx45Ofn1+iZeZ0WSRMTE3Tv3h0nT55Utr3xxhs4ffo0UlJSalUkKzqSdHV1xZ07d9Q2zMOHD3Ht2jW0adOmyuub5eP+GcoYsIaWL1D/OT98+BBXr16Fq6urVq6na0NZWRkOHz6MgQMHGsSD5vWRb3J2MobGDq1yuX3j9zXYkeST8ZgbmePLzl9iysUpeKB40ODx6EJF/VxQUAA7O7saF0mdnm51dnZGx44dVdp8fHzw7bffAgCcnJwAALdu3VIpkrdu3UK3bt0qXKepqWmFp7KMjY3V/lPI5XJIJBIYGRlVeY1IoVAAgHL5ps7Q8gXqP2cjIyNIJJIK90Vd08eY6pM28w3yCEJLy5a4XnC9wuuAEkjgInNBkEdQg1wD1BTPA8UDPFA8aPB4dOnJfq5tf+v00693797IyMhQafv999/h7u4O4PFNPE5OTjhy5IhyfkFBAU6dOoWAgIAGjZUazokTJ9ClSxcYGxtj5MiR1XpNVFSUxj+caksikWDPnj3VXr4+YiD9JzWSIiY0BsDjgvik8umVoSsbrCDpWzyNnU6L5OzZs5GamorFixfjypUriI2Nxeeff46IiAgAjz+kZs2ahY8++gjfffcdLly4gFdeeQWtWrWq9odnU3T79m3885//hJubG0xNTeHk5ISQkBCcOHFC16FpxZw5c9CtWzdkZWVh8+bNdV5fVFQUJBJJpT8VycnJweDBg+v8/tT0hfmEIW5sHFrLWqu0u8hcdPK4hb7F05jp9HRrjx49sHv3bsyfPx+LFi2Ch4cHVq5ciQkTJiiXefvtt3H//n1Mnz4d9+7dQ58+fXDgwAG9uaYDPL7lOik7CTmFOXC2ckagW2C9/pUWHh6O0tJSbNmyBW3btsWtW7dw5MgR5OXl1dt7NqTMzEzMmDGjRl9nU5m33noLM2bMUE736NED06dPV95F/bTS0lKYmZkpT/cTVUeYTxhGeI1o0M+C6sSTmJWIgosF2Dd+n0GcYtU60cTl5+cLACI/P19t3oMHD8SlS5fEgwcPqlyPXC4Xd+/eFXK5XKX920vfCpdPXQSioPxx+dRFfHvpW63l8KS7d+8KAOL48eMal8nKyhIAxNmzZ9Ved+zYMWXbxYsXxdChQ4WVlZWwtLQUffr0EVeuXBFCPM539erVomPHjsLExEQ4OTmJiIgIlfVNnTpV2NnZCSsrK9GvXz9x7tw55fxz586J4OBgYWlpKaysrISfn584ffq0EEKIq1evimHDhgkbGxthYWEhOnbsKPbt26eM+8mfTZs2iU2bNglra2uVHHfv3i2e3H0XLlwounbtWq1t6O7uLlasWKGc7tu3r4iIiBBvvPGGsLW1FcHBwUIIIQCI3bt3K5d7++23haenpzA3NxceHh7i/fffF6WlpdWOoSb7W0MpLS0Ve/bsUcmjKTO0fIVgzuUqqwWVMYw7MupJ+agWTw5DBfxvVIv49Hitv6elpSUsLS2xZ88elbt4a+r69esICgqCqakpjh49irS0NEyZMkX5aMK6deswd+5cTJs2DRcuXMB3332H9u3bK18/ZswY5ObmYv/+/UhLS4Ofnx+ef/55/P333wCACRMmwMXFBadPn0ZaWhrmzZunvHAeERGBkpISJCYm4sKFC1i2bBksLS3h6uqKnJwcyGQyrFy5Ejk5OXjxxRfrsLWqb8uWLTAxMcGBAwewdu3aCpexsrLC5s2bcenSJcTExGDDhg1YsWJFg8RHRLqh87FbGytdjWrRrFkzbN68GdOmTcP69evh5+eHvn37Yty4cfD19a32etasWQNra2vs2LFDWbw6dOignL948WJERETgjTfeUN7p2aNHDwBAcnIyfvrpJ+Tm5irvJP7Xv/6FPXv2IC4uDtOnT0d2djbmzp0Lb29vAFAZazc7Oxvh4eHo0qULAKBt27bKeU5OTpBIJLC2tm7Q052enp5YtmwZCgoKNN4e/v777yt/b9OmDd566y3s2LEDb7/9dkOFSUQNjEeStZSUnaR2BPkkAYFrBdeQlJ2k9fcODw/HjRs38N133yE0NBTHjx+Hn59fjW5yOXfuHAIDAyu8LTo3Nxc3btxA3759K3ztL7/8gqKiIrRs2VJ5ZGtpaYmsrCxkZmYCeHzzzauvvooBAwZg6dKlynbg8bOwH330EXr37o2FCxfi/PnzNdsA9cDf37/KZb755hv07t0bTk5OsLS0xPvvv4/s7OwGiI6IdIVFspZyCnO0ulxNmZmZYeDAgViwYAFOnjyJSZMmYeHChQCgPPITT4wT8fSQTObmmofQqmweABQVFcHZ2Rnnzp1T+cnIyMDcuXMBPL6j9Ndff8XQoUNx9OhRdOzYEbt37wYAvPrqq/jjjz/w8ssv48KFC+jevTtWr16t8f2MjIxUcqkon7pq3rx5pfNTUlIwYcIEDBkyBN9//z3Onj2L9957D6WlpVqNg4j0C4tkLTlbOVe9UA2Wq6uOHTvi/v37AAB7e3sAjx9hKHfu3DmV5X19fZGUlFRhsbGyskKbNm2QkJBQ4Xv5+fnh5s2baNasGdq3b6/yY2dnp1yuQ4cOmD17Ng4dOoSwsDCVgexdXV0xY8YMxMfH480338SGDRs05mZvb4/CwkJlfhXlU99OnjwJd3d3vPfee+jevTs8PT3x559/NmgMRNTwWCRrKdAtEC4yF7WHdctJIIGrzBWBboFafd+8vDz0798f27Ztw/nz55GVlYVdu3Zh+fLlym9KMTc3x7PPPoulS5ciPT0dCQkJKtfTAGDmzJkoKCjAuHHj8PPPP+Py5cvYunWrcnCHDz74AGvWrMHq1atx+fJlnDlzRnm0N2DAAAQEBGDkyJE4dOgQrl69ipMnT+K9997Dzz//jAcPHmDmzJk4fvw4/vzzT5w4cQKnT5+Gj48PAGDWrFk4ePAgsrKycObMGRw7dkw5ryK9evWChYUF3n33XWRmZiI2NlYrz0/WhKenJ7Kzs7Fjxw5kZmZi1apVyiNjImq6WCRrSVejWlhaWqJXr15YsWIFgoKC0LlzZyxYsADTpk3DZ599plzuyy+/xKNHj+Dv768ckOFJLVu2xNGjR1FUVIS+ffvC398fGzZsUF6jnDhxIhYvXox169ahU6dOGDZsGC5fvvw4P4kEP/zwA4KCgjB58mR06NAB48aNw59//glHR0dIpVLk5eXhlVdeQYcOHTB27FgMHjxYOfC8XC5HREQEfHx8EBoaig4dOmi8oxQAbG1tsW3bNvzwww/o0qULtm/fjqioKK1u16q88MILmD17NmbOnIlu3brh5MmTWLBgQYPGQEQNT6cDnDeEgoICWFtbVzio7cOHD5GVlQUPD48qBydQKBTKOx+fHNezom8jd5W5YmXoykY9qoWmfJuy+s65JvtbQykrK8MPP/yAIUOGGMTYrYaWL8CcnxzgXFMtqAwfAakjfRtlg4iItIdFUgukRlIEtwnWdRhERHXS0ENsNgYskkREVOGlIxeZC2JCYxr1paO6MoyLTUREpJEuhthsLFgkAbUH1YnqA/cz0kdVDbEJALMOzIJcIW/o0PSCQRfJ8rueiouLdRwJGYLy/cxQ7jCkxkGXQ2w2BgZ9TVIqlcLGxga5ubkAAAsLC41fwKtQKFBaWoqHDx8axCMRhpYvUH85CyFQXFyM3Nxc2NjYQCo17BshSL/oeohNfWfQRRKA8psmygulJkIIPHjwAObm5hoLaVNiaPkC9Z+zjY0Nv8iZ9I6+DbGpbwy+SEokEjg7O8PBwaHSQbPLysqQmJiIoKAggzhdZmj5AvWbs7GxMY8gSS+VD7F5veB6hdclJZDAReai9SE2GwuDL5LlpFJppR9iUqkUjx49gpmZmUEUDUPLFzDMnInKh9gcvXM0JJCoFMr6HGKzsTCMi01ERKRRmE8Y4sbGobWstUq7i8wFcWPjDPo5SR5JEhERh9jUgEWSiIgAcIjNivB0KxERkQYskkRERBqwSBIREWnAIklERKQBiyQREZEGLJJEREQasEgSERFpwCJJRESkAYskERGRBhxxh3RCrpDXaPirmi5viPRtG+lbPPqI20j/sUhSg4tPj0fkgUiVb0N3kbkgJjSmwoGUa7q8IdK3baRv8egjbqPGgadbqUHFp8dj9M7RKh8MAHC94DpG7xyN+PT4Oi1viPRtG+lbPPqI26jxYJGkBiNXyBF5ILLCL3Ytb5t1YBbkCnmtljdE+raN9C0efcRt1LiwSFKDScpOUvvL+UkCAtcKriEpO6lWyxsifdtG+haPPuI2alxYJKnB5BTm1Gi5mi5viPRtG+lbPPqI26hxYZGkBuNs5Vyj5Wq6vCHSt22kb/HoI26jxoVFkhpMoFsgXGQukEBS4XwJJHCVuSLQLbBWyxsifdtG+haPPuI2alxYJKnBSI2kiAmNAQC1D4jy6ZWhK5XPidV0eUOkb9tI3+LRR9xGjQuLJDWoMJ8wxI2NQ2tZa5V2F5kL4sbGqT0fVtPlDZG+bSN9i0cfcRs1HhxMgBpcmE8YRniNqPZIIzVd3hDp2zbSt3j0EbdR48AiSTohNZIiuE1wvS1viPRtG+lbPPqI20j/6fR0a1RUFCQSicqPt7e3cv7Dhw8RERGBli1bwtLSEuHh4bh165YOIyYiIkOi82uSnTp1Qk5OjvInOTlZOW/27NnYu3cvdu3ahYSEBNy4cQNhYTxXT0REDUPnp1ubNWsGJycntfb8/Hxs3LgRsbGx6N+/PwBg06ZN8PHxQWpqKp599tmGDpWIiAyMzovk5cuX0apVK5iZmSEgIABLliyBm5sb0tLSUFZWhgEDBiiX9fb2hpubG1JSUjQWyZKSEpSUlCinCwoKAABlZWUoKyurdZzlr63LOhoTQ8sXYM6GwNDyBZjz0201JRFCqI+y20D279+PoqIieHl5IScnB9HR0bh+/TouXryIvXv3YvLkySoFDwB69uyJfv36YdmyZRWuMyoqCtHR0WrtsbGxsLCwqJc8iIhIvxUXF2P8+PHIz8+HTCar9ut0WiSfdu/ePbi7u+PTTz+Fubl5rYpkRUeSrq6uuHPnTo02zNPKyspw+PBhDBw4EMbGxrVeT2NhaPkCzNkQcja0fAHmXJ5zQUEB7OzsalwkdX669Uk2Njbo0KEDrly5goEDB6K0tBT37t2DjY2Ncplbt25VeA2znKmpKUxNTdXajY2NtbKDaGs9jYWh5QswZ0NgaPkCzLm2uev87tYnFRUVITMzE87OzvD394exsTGOHDminJ+RkYHs7GwEBAToMEoiIjIUOj2SfOuttzB8+HC4u7vjxo0bWLhwIaRSKV566SVYW1tj6tSpmDNnDmxtbSGTyfD6668jICCAd7YSEVGD0GmR/Ouvv/DSSy8hLy8P9vb26NOnD1JTU2Fvbw8AWLFiBYyMjBAeHo6SkhKEhIRg7dq1ugxZL8gVcg5lRXUmV8iRnP34ueTk7GQEeQRxPyJ6ik6L5I4dOyqdb2ZmhjVr1mDNmjUNFJH+i0+PR+SBSJVvNneRuSAmNIaDIlO1le9HeUV52O67HUNjh6KlZUvuR0RP0atrklS5+PR4jN45WqVAAsD1gusYvXM04tPjdRQZNSbcj4iqj0WykZAr5Ig8EAkB9Sd2yttmHZgFuULe0KFRI8L9iKhmWCQbiaTsJLW//J8kIHCt4BqSspMaMCpqbLgfEdUMi2QjkVOYo9XlyDBxPyKqGRbJRsLZylmry5Fh4n5EVDMsko1EoFsgXGQukEBS4XwJJHCVuSLQLbCBI6PGhPsRUc2wSDYSUiMpYkJjAEDtA658emXoSj7nRpXifkRUMyySjUiYTxjixsahtay1SruLzAVxY+P4fBtVi6HuR08PnsA7eKk69GqAc6pamE8YRniN4Ig7VCfl+1FiViIKLhZg3/h9TXrEHQ6eQLXFI8lGSGokRXCbYLzU5SUEtwlush9sVL+kRlL0cesDAOjj1qfJ7kccPIHqgkWSiJosDp5AdcUiSURNFgdPoLpikSSiJouDJ1BdsUgSUZPFwROorlgkiajJ4uAJVFcskkTUZHHwBKorFkkiatIMdfAE0g4OJkBETZ6hDZ5A2sMjSSIyCIYyeAJpF48k9YBcIecwc6SC+wSRfmCRrIanB0bW5mma8jEln3zg2UXmwjElDRj3CSL9wdOtVYhPj0ebmDYYGjsUADA0dijaxLTRyniPHFOSnsZ9gki/sEhWoj4/sDimJD2N+wSR/mGR1KC+P7A4piQ9jfsEkf5hkdSgvj+wOKYkPY37BJH+YZHUoL4/sDimJD2N+wSR/mGR1KC+P7A4piQ9jfsEkf5hkdSgvj+wOKakfnn6MR9d3BzDfYJI/9SpSJaWliIjIwOPHj3SVjx6oyE+sDimpH6oz8d8aor7BJF+qdVgAsXFxXj99dexZcsWAMDvv/+Otm3b4vXXX0fr1q0xb948rQapK+UfWJEHIpFXlKdsd5G5YGXoSq18YJWPKcnRVXSj/DEfAQFzI3Nle/ljProoTNwniPRHrYrk/Pnz8csvv+D48eMIDQ1Vtg8YMABRUVFNpkgCDTMwstRIiuA2wVpbH1VPVY/5SCDBrAOzMMJrRIMXKO4TRPqhVqdb9+zZg88++wx9+vSBRPK/U5GdOnVCZmam1oLTFxwYuWnic4lEVJVaFcnbt2/DwcFBrf3+/fsqRZNIn/G5RCKqSq2KZPfu3bFv3z7ldHlh/OKLLxAQEKCdyIjqGZ9LJKKq1Oqa5OLFizF48GBcunQJjx49QkxMDC5duoSTJ08iISFB2zES1Yvyx3yuF1yv8LqkBBK4yFz4XCKRAavVkWSfPn3wyy+/4NGjR+jSpQsOHToEBwcHpKSkwN/fX9sxEtULPpdIRFWpcZEsKyvDlClTIJFIsGHDBvz000+4dOkStm3bhi5dutRHjET1hs8lElFlalwkjY2N8e2339ZHLEQ6EeYThquRV7Fv/OPr7PvG70NWZBYLJBHV7nTryJEjsWfPHi2HQqQ7fMyHiCpSqxt3PD09sWjRIpw4cQL+/v5o3ry5yvw33nhDK8ERERHpUq2K5MaNG2FjY4O0tDSkpaWpzJNIJAZfJOUKuUENKaaP+epjTFQ5fewzfYyJGlatimRWVpa248DSpUsxf/58REZGYuXKlQCAhw8f4s0338SOHTtQUlKCkJAQrF27Fo6Ojlp/f22JT49H5IFIlZFcXGQuiAmNaZLXuPQxX32MiSqnj32mjzFRw6vzV2UJISCE+jNmNXH69Gn85z//ga+vr0r77NmzsXfvXuzatQsJCQm4ceMGwsL0d+csHyz76aHOygfL1sW3StQnfcxXH2Oiyuljn+ljTKQbtS6SX331Fbp06QJzc3OYm5vD19cXW7durfF6ioqKMGHCBGzYsAEtWrRQtufn52Pjxo349NNP0b9/f/j7+2PTpk04efIkUlNTaxt2valqsGwAmHVglk6+p7A+6GO++hgTVU4f+0wfYyLdqdXp1k8//RQLFizAzJkz0bt3bwBAcnIyZsyYgTt37mD27NnVXldERASGDh2KAQMG4KOPPlK2p6WloaysDAMGDFC2eXt7w83NDSkpKXj22WcrXF9JSQlKSkqU0wUFBQAeP99ZVlZWozyfVP5aTetIzk5GXlGeytctPe1O0R0kZiUq76LUZ40x37rGVFXOTZGuc27o/ag6+erjvl0Xuu5jXago59rmLxG1OFfq4eGB6OhovPLKKyrtW7ZsQVRUVLWvWe7YsQMff/wxTp8+DTMzMwQHB6Nbt25YuXIlYmNjMXnyZJWCBwA9e/ZEv379sGzZsgrXGRUVhejoaLX22NhYWFhYVDNDIiJqSoqLizF+/Hjk5+dDJpNV+3W1OpLMycnBc889p9b+3HPPISenet+YcO3aNURGRuLw4cMwMzOrTRgVmj9/PubMmaOcLigogKurKwYNGlSjDfO0srIyHD58GAMHDoSxsbHa/OTsZOU321dm3/h9jeavz8aWb11jqirnpkjXOTf0flSdfPVx364LXfexLlSUc/lZxZqqVZFs3749du7ciXfffVel/ZtvvoGnp2e11pGWlobc3Fz4+fkp2+RyORITE/HZZ5/h4MGDKC0txb1792BjY6Nc5tatW3ByctK4XlNTU5iamqq1Gxsba2UH0bSeII8gtLRsWeVg2dr+wub61pjy1VZM2tpXGhNd5ayr/aiyfPVx39YGQ9+va5t7rW7ciY6OxgcffIDQ0FB8+OGH+PDDDxEaGoro6GgsWrSoWut4/vnnceHCBZw7d0750717d0yYMEH5u7GxMY4cOaJ8TUZGBrKzs/Xy67gMbbBsfcxXH2Oiyuljn+ljTKQ7tSqS4eHhOHXqFOzs7LBnzx7s2bMHdnZ2+OmnnzBq1KhqrcPKygqdO3dW+WnevDlatmyJzp07w9raGlOnTsWcOXNw7NgxpKWlYfLkyQgICNB4046uGdpg2fqYrz7GRJXTxz7Tx5hIN2p1uhUA/P39sW3bNm3GombFihUwMjJCeHi4ymAC+izMJwwjvEYYzCgd+pivPsZEldPHPtPHmKjh1apI/vDDD5BKpQgJCVFpP3jwIBQKBQYPHlyrYI4fP64ybWZmhjVr1mDNmjW1Wp+uSI2kCG4TrOswGow+5quPMVHl9LHP9DEmali1Ot06b948yOXqD9IKITBv3rw6B0VERKQPalUkL1++jI4dO6q1e3t748qVK3UOioiISB/UqkhaW1vjjz/+UGu/cuWK2tdmERERNVa1KpIjRozArFmzkJmZqWy7cuUK3nzzTbzwwgtaC46IiEiXalUkly9fjubNm8Pb2xseHh7w8PCAt7c3WrZsiX/961/ajpGIiEgnanV3q7W1NU6ePInDhw/jl19+gbm5Obp27YrAwEBtx0dERKQzNTqSTElJwffffw8AkEgkGDRoEBwcHPCvf/0L4eHhmD59utqA5ERERI1VjYrkokWL8OuvvyqnL1y4gGnTpmHgwIGYN28e9u7diyVLlmg9SCIiIl2oUZE8d+4cnn/+eeX0jh070LNnT2zYsAFz5szBqlWrsHPnTq0HSUREpAs1uiZ59+5dODo6KqcTEhJURtfp0aMHrl27pr3oSCfkCjmSs5MBPP7aoMb2bQf6SK6QN/rhzfQtB32Lh6qnpv2m636uUZF0dHREVlYWXF1dUVpaijNnzqh8wXFhYaHBfRVLUxOfHo/IA5HIK8rDdt/tGBo7FC0tWyImNIaDOtdS+Tb9q+AvZZuLzKVRbVN9y0Hf4qHqqWm/6UM/1+h065AhQzBv3jwkJSVh/vz5sLCwULmj9fz582jXrp3Wg6SGEZ8ej9E7R6vskABwveA6Ru8cjfj0eB1F1ng1hW2qbznoWzxUPTXtN33p5xoVyQ8//BDNmjVD3759sWHDBmzYsAEmJibK+V9++SUGDRqk9SCp/skVckQeiKzwS2bL22YdmAW5Qn3MXqpYU9im+paDvsVD1VPTftOnfq5RkbSzs0NiYiLu3r2Lu3fvqn135K5du7Bw4UKtBkgNIyk7Se0vticJCFwruIak7KQGjKpxawrbVN9y0Ld4qHpq2m/61M+1HkygIra2tnUKhnQnpzBHq8tR09im+paDvsVD1VPTftOnfq7VsHTU9DhbOWt1OWoa21TfctC3eKh6atpv+tTPLJIEAAh0C4SLzAUSSCqcL4EErjJXBLpx6MHqagrbVN9y0Ld4qHpq2m/61M8skgTg8Tewx4TGAIDajlk+vTJ0JZ9Dq4GmsE31LQd9i4eqp6b9pk/9zCJJSmE+YYgbG4fWstYq7S4yF8SNjePzZ7XQFLapvuWgb/FQ9dS03/Sln2t14w41XWE+YRjhNQKJWYkouFiAfeP3ccSdOirfpo15dBh9y0Hf4qHqqWm/6UM/s0iSGqmRFH3c+uCHiz+gj1sffvBogdRIiuA2wboOo070LQd9i4eqp6b9put+5ulWIiIiDVgkiYiINGCRJCIi0oBFkoiISAMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkkiIiINWCSJiIg0YJEkIiLSgEWSiIhIAxZJIiIiDVgkiYiINGCRJCIi0oBFkoiISAMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkkiIiINdFok161bB19fX8hkMshkMgQEBGD//v3K+Q8fPkRERARatmwJS0tLhIeH49atWzqMmIiIDIlOi6SLiwuWLl2KtLQ0/Pzzz+jfvz9GjBiBX3/9FQAwe/Zs7N27F7t27UJCQgJu3LiBsLAwXYZMREQGpJku33z48OEq0x9//DHWrVuH1NRUuLi4YOPGjYiNjUX//v0BAJs2bYKPjw9SU1Px7LPP6iJkIiIyIDotkk+Sy+XYtWsX7t+/j4CAAKSlpaGsrAwDBgxQLuPt7Q03NzekpKRoLJIlJSUoKSlRThcUFAAAysrKUFZWVuv4yl9bl3U0JoaWL8CcDYGh5Qsw56fbakrnRfLChQsICAjAw4cPYWlpid27d6Njx444d+4cTExMYGNjo7K8o6Mjbt68qXF9S5YsQXR0tFr7oUOHYGFhUed4Dx8+XOd1NCaGli/AnA2BoeULMOfi4uJarUPnRdLLywvnzp1Dfn4+4uLiMHHiRCQkJNR6ffPnz8ecOXOU0wUFBXB1dcWgQYMgk8lqvd6ysjIcPnwYAwcOhLGxca3X01gYWr4AczaEnA0tX4A5l+dcflaxpnReJE1MTNC+fXsAgL+/P06fPo2YmBi8+OKLKC0txb1791SOJm/dugUnJyeN6zM1NYWpqalau7GxsVZ2EG2tp7EwtHwB5mwIDC1fgDnXNne9e05SoVCgpKQE/v7+MDY2xpEjR5TzMjIykJ2djYCAAB1GSEREhkKnR5Lz58/H4MGD4ebmhsLCQsTGxuL48eM4ePAgrK2tMXXqVMyZMwe2traQyWR4/fXXERAQwDtbiYioQei0SObm5uKVV15BTk4OrK2t4evri4MHD2LgwIEAgBUrVsDIyAjh4eEoKSlBSEgI1q5dq8uQiYjo/5Mr5EjKTkJOYQ6crZwR6BYIqZFU12FplU6L5MaNGyudb2ZmhjVr1mDNmjUNFBEREVVHfHo8Ig9E4q+Cv5RtLjIXxITGIMyn6Qz6onfXJImISL/Fp8dj9M7RKgUSAK4XXMfonaMRnx6vo8i0j0WSiIiqTa6QI/JAJASE2rzytlkHZkGukDd0aPWCRZKIiKotKTtJ7QjySQIC1wquISk7qQGjqj8skkREVG05hTlaXU7fsUgSEVG1OVs5a3U5fcciSURE1RboFggXmQskkFQ4XwIJXGWuCHQLbODI6geLJBERVZvUSIqY0BgAUCuU5dMrQ1c2meclWSSJiKhGwnzCEDc2Dq1lrVXaXWQuiBsb16Sek9T5AOdERNT4hPmEYYTXCI64Q0REVBGpkRTBbYJ1HUa94ulWIiIiDVgkiYiINGCRJCIi0oBFkoiISAMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkkiIiINOOKOAZAr5E1+6CgiovrAItnExafHI/JApMo3ibvIXBATGtOkBiEmIqoPPN3ahMWnx2P0ztEqBRIArhdcx+idoxGfHq+jyIiIGgcWySZKrpAj8kAkBITavPK2WQdmQa6QN3RoRESNBotkE5WUnaR2BPkkAYFrBdeQlJ3UgFERETUuLJJNVE5hjlaXIyIyRCySTZSzlbNWlyMiMkQskk1UoFsgXGQukEBS4XwJJHCVuSLQLbCBIyMiajxYJJsoqZEUMaExAKBWKMunV4au5POSRESVYJFswsJ8whA3Ng6tZa1V2l1kLogbG8fnJImIqsDBBJq4MJ8wjPAawRF3iIhqgUXSAEiNpAhuE6zrMIiIGh2ebiUiItKARZKIiEgDFkkiIiINWCSJiIg0YJEkIiLSgEWSiIhIAxZJIiIiDVgkiYiINGCRJCIi0oAj7hDVglwh51B/1OC43zU8nR5JLlmyBD169ICVlRUcHBwwcuRIZGRkqCzz8OFDREREoGXLlrC0tER4eDhu3bqlo4iJgPj0eLSJaYN+W/phfPx49NvSD21i2iA+PV7XoVETxv1ON3RaJBMSEhAREYHU1FQcPnwYZWVlGDRoEO7fv69cZvbs2di7dy927dqFhIQE3LhxA2Fh/PYK0o349HiM3jkafxX8pdJ+veA6Ru8czQ8sqhfc73RHp6dbDxw4oDK9efNmODg4IC0tDUFBQcjPz8fGjRsRGxuL/v37AwA2bdoEHx8fpKam4tlnn9VF2GSg5Ao5Ig9EQkCozRMQkECCWQdmYYTXCJ4CI63hfqdbenVNMj8/HwBga2sLAEhLS0NZWRkGDBigXMbb2xtubm5ISUmpsEiWlJSgpKREOV1QUAAAKCsrQ1lZWa1jK39tXdbRmBhavkDVOSdnJyOvKA/mRuYa13Gn6A4SsxLRx61PvcSobYbWz40x37rud40x57qqKOfa5i8RQqj/eaIDCoUCL7zwAu7du4fk5GQAQGxsLCZPnqxS9ACgZ8+e6NevH5YtW6a2nqioKERHR6u1x8bGwsLCon6CJyIivVZcXIzx48cjPz8fMpms2q/TmyPJiIgIXLx4UVkga2v+/PmYM2eOcrqgoACurq4YNGhQjTbM08rKynD48GEMHDgQxsbGdYqxMTC0fIGqc07OTsbQ2KFVrmff+H2N6kjSkPq5MeZb1/2uMeZcVxXlXH5Wsab0okjOnDkT33//PRITE+Hi4qJsd3JyQmlpKe7duwcbGxtl+61bt+Dk5FThukxNTWFqaqrWbmxsrJUdRFvraSwMLV9Ac85BHkFoadkS1wuuV3h9SAIJXGQuCPIIanTXhgytnxtTvtra7xpTztryZM61zV2nd7cKITBz5kzs3r0bR48ehYeHh8p8f39/GBsb48iRI8q2jIwMZGdnIyAgoKHDJQMnNZIiJjQGwOMPpieVT68MXdnoCiTpN+53uqXTIhkREYFt27YhNjYWVlZWuHnzJm7evIkHDx4AAKytrTF16lTMmTMHx44dQ1paGiZPnoyAgADe2Uo6EeYThrixcWgta63S7iJzQdzYOIT58PEk0j7ud7qj09Ot69atAwAEBwertG/atAmTJk0CAKxYsQJGRkYIDw9HSUkJQkJCsHbt2gaOlOh/wnzCMMJrBEc+oQbF/U43dFokq3NjrZmZGdasWYM1a9Y0QERE1SM1kiK4TbCuwyADw/2u4XGAcyIiIg1YJImIiDRgkSQiItJAL56TrE/l1z1r+yBpubKyMhQXF6OgoMAgnjUytHwB5mwIORtavgBzfnowgZoOMtfki2RhYSEAwNXVVceREBGRrhUWFsLa2rray+vN2K31RaFQ4MaNG7CysoJEIqn6BRqUD2937dq1Og1v11gYWr4AczaEnA0tX4A5l+cshEBhYSFatWoFI6PqX2ls8keSRkZGKkPd1ZVMJjOYHQ0wvHwB5mwIDC1fgDkDqNERZDneuENERKQBiyQREZEGLJLVZGpqioULF1b4DSNNkaHlCzBnQ2Bo+QLMua6a/I07REREtcUjSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFslqWLNmDdq0aQMzMzP06tULP/30k65DqjdRUVGQSCQqP97e3roOS6sSExMxfPhwtGrVChKJBHv27FGZL4TABx98AGdnZ5ibm2PAgAG4fPmyboLVgqrynTRpklqfh4aG6iZYLVmyZAl69OgBKysrODg4YOTIkcjIyFBZ5uHDh4iIiEDLli1haWmJ8PBw3Lp1S0cR10118g0ODlbr5xkzZugo4rpbt24dfH19lQMGBAQEYP/+/cr52upfFskqfPPNN5gzZw4WLlyIM2fOoGvXrggJCUFubq6uQ6s3nTp1Qk5OjvInOTlZ1yFp1f3799G1a1eNX+S9fPlyrFq1CuvXr8epU6fQvHlzhISE4OHDhw0cqXZUlS8AhIaGqvT59u3bGzBC7UtISEBERARSU1Nx+PBhlJWVYdCgQbh//75ymdmzZ2Pv3r3YtWsXEhIScOPGDYSFhekw6tqrTr4AMG3aNJV+Xr58uY4irjsXFxcsXboUaWlp+Pnnn9G/f3+MGDECv/76KwAt9q+gSvXs2VNEREQop+VyuWjVqpVYsmSJDqOqPwsXLhRdu3bVdRgNBoDYvXu3clqhUAgnJyfxySefKNvu3bsnTE1Nxfbt23UQoXY9na8QQkycOFGMGDFCJ/E0lNzcXAFAJCQkCCEe96mxsbHYtWuXcpn09HQBQKSkpOgqTK15Ol8hhOjbt6+IjIzUXVANoEWLFuKLL77Qav/ySLISpaWlSEtLw4ABA5RtRkZGGDBgAFJSUnQYWf26fPkyWrVqhbZt22LChAnIzs7WdUgNJisrCzdv3lTpc2tra/Tq1atJ9/nx48fh4OAALy8v/POf/0ReXp6uQ9Kq/Px8AICtrS0AIC0tDWVlZSr97O3tDTc3tybRz0/nW+7rr7+GnZ0dOnfujPnz56O4uFgX4WmdXC7Hjh07cP/+fQQEBGi1f5v8AOd1cefOHcjlcjg6Oqq0Ozo64rffftNRVPWrV69e2Lx5M7y8vJCTk4Po6GgEBgbi4sWLsLKy0nV49e7mzZsAUGGfl89rakJDQxEWFgYPDw9kZmbi3XffxeDBg5GSkgKpVKrr8OpMoVBg1qxZ6N27Nzp37gzgcT+bmJjAxsZGZdmm0M8V5QsA48ePh7u7O1q1aoXz58/jnXfeQUZGBuLj43UYbd1cuHABAQEBePjwISwtLbF792507NgR586d01r/skiSisGDByt/9/X1Ra9eveDu7o6dO3di6tSpOoyM6su4ceOUv3fp0gW+vr5o164djh8/jueff16HkWlHREQELl682OSurWuiKd/p06crf+/SpQucnZ3x/PPPIzMzE+3atWvoMLXCy8sL586dQ35+PuLi4jBx4kQkJCRo9T14urUSdnZ2kEqlandE3bp1C05OTjqKqmHZ2NigQ4cOuHLliq5DaRDl/WrIfd62bVvY2dk1iT6fOXMmvv/+exw7dkzlK/OcnJxQWlqKe/fuqSzf2PtZU74V6dWrFwA06n42MTFB+/bt4e/vjyVLlqBr166IiYnRav+ySFbCxMQE/v7+OHLkiLJNoVDgyJEjCAgI0GFkDaeoqAiZmZlwdnbWdSgNwsPDA05OTip9XlBQgFOnThlMn//111/Iy8tr1H0uhMDMmTOxe/duHD16FB4eHirz/f39YWxsrNLPGRkZyM7ObpT9XFW+FTl37hwANOp+fppCoUBJSYl2+1e79xY1PTt27BCmpqZi8+bN4tKlS2L69OnCxsZG3Lx5U9eh1Ys333xTHD9+XGRlZYkTJ06IAQMGCDs7O5Gbm6vr0LSmsLBQnD17Vpw9e1YAEJ9++qk4e/as+PPPP4UQQixdulTY2NiI//73v+L8+fNixIgRwsPDQzx48EDHkddOZfkWFhaKt956S6SkpIisrCzx448/Cj8/P+Hp6SkePnyo69Br7Z///KewtrYWx48fFzk5Ocqf4uJi5TIzZswQbm5u4ujRo+Lnn38WAQEBIiAgQIdR115V+V65ckUsWrRI/PzzzyIrK0v897//FW3bthVBQUE6jrz25s2bJxISEkRWVpY4f/68mDdvnpBIJOLQoUNCCO31L4tkNaxevVq4ubkJExMT0bNnT5GamqrrkOrNiy++KJydnYWJiYlo3bq1ePHFF8WVK1d0HZZWHTt2TABQ+5k4caIQ4vFjIAsWLBCOjo7C1NRUPP/88yIjI0O3QddBZfkWFxeLQYMGCXt7e2FsbCzc3d3FtGnTGv0fgRXlC0Bs2rRJucyDBw/Ea6+9Jlq0aCEsLCzEqFGjRE5Oju6CroOq8s3OzhZBQUHC1tZWmJqaivbt24u5c+eK/Px83QZeB1OmTBHu7u7CxMRE2Nvbi+eff15ZIIXQXv/yq7KIiIg04DVJIiIiDVgkiYiINGCRJCIi0oBFkoiISAMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkmiBhIVFYVu3brV6DUSiQR79uypl3i04erVq5BIJMpxQImaGhZJolqQSCSV/kRFRam95q233lIZcFkbJk2aBIlEgqVLl6q079mzBxKJRKvvRWSI+H2SRLWQk5Oj/P2bb77BBx98gIyMDGWbpaWl8nchBORyOSwtLVXatcXMzAzLli3D//3f/6FFixZaX78ulJaWwsTERNdhEPFIkqg2nJyclD/W1taQSCTK6d9++w1WVlbYv38//P39YWpqiuTkZLXTradPn8bAgQNhZ2cHa2tr9O3bF2fOnKlxLAMGDICTkxOWLFmicZmKTvWuXLkSbdq0UU5PmjQJI0eOxOLFi+Ho6AgbGxssWrQIjx49wty5c2FrawsXFxds2rRJbf2//fYbnnvuOZiZmaFz585qX3x78eJFDB48GJaWlnB0dMTLL7+MO3fuKOcHBwdj5syZmDVrFuzs7BASElLj7UBUH1gkierJvHnzsHTpUqSnp8PX11dtfmFhISZOnIjk5GSkpqbC09MTQ4YMQWFhYY3eRyqVYvHixVi9ejX++uuvOsV89OhR3LhxA4mJifj000+xcOFCDBs2DC1atMCpU6cwY8YM/N///Z/a+8ydOxdvvvkmzp49i4CAAAwfPhx5eXkAgHv37qF///545pln8PPPP+PAgQO4desWxo4dq7KOLVu2wMTEBCdOnMD69evrlAeRtrBIEtWTRYsWYeDAgWjXrh1sbW3V5vfv3x//+Mc/4O3tDR8fH3z++ecoLi5WOwqrjlGjRqFbt25YuHBhnWK2tbXFqlWr4OXlhSlTpsDLywvFxcV499134enpifnz58PExATJyckqr5s5cybCw8Ph4+ODdevWwdraGhs3bgQAfPbZZ3jmmWewePFieHt745lnnsGXX36JY8eO4ffff1euw9PTE8uXL4eXlxe8vLzqlAeRtrBIEtWT7t27Vzr/1q1bmDZtGjw9PWFtbQ2ZTIaioiJkZ2fX6v2WLVuGLVu2ID09vVavB4BOnTrByOh/HwuOjo7o0qWLcloqlaJly5bIzc1Ved2T3/berFkzdO/eXRnHL7/8gmPHjimvyVpaWsLb2xsAkJmZqXydv79/reMmqi+8cYeonjRv3rzS+RMnTkReXh5iYmLg7u4OU1NTBAQEoLS0tFbvFxQUhJCQEMyfPx+TJk1SmWdkZISnvzq2rKxMbR3GxsYq0xKJpMI2hUJR7biKioowfPhwLFu2TG2es7Oz8veqtheRLrBIEunIiRMnsHbtWgwZMgQAcO3aNZWbWWpj6dKl6Natm9rpSnt7e9y8eRNCCOWjIdp8tjE1NRVBQUEAgEePHiEtLQ0zZ84EAPj5+eHbb79FmzZt0KwZP3KoceHpViId8fT0xNatW5Geno5Tp05hwoQJMDc3r9M6u3TpggkTJmDVqlUq7cHBwbh9+zaWL1+OzMxMrFmzBvv376/Tez1pzZo12L17N3777TdERETg7t27mDJlCgAgIiICf//9N1566SWcPn0amZmZOHjwICZPngy5XK61GIjqA4skkY5s3LgRd+/ehZ+fH15++WW88cYbcHBwqPN6Fy1apHY61MfHB2vXrsWaNWvQtWtX/PTTT3jrrbfq/F7lli5diqVLl6Jr165ITk7Gd999Bzs7OwBAq1atcOLECcjlcgwaNAhdunTBrFmzYGNjo3L9k0gfScTTFyqIiIgIAI8kiYiINGKRJCIi0oBFkoiISAMWSSIiIg1YJImIiDRgkSQiItKARZKIiEgDFkkiIiINWCSJiIg0YJEkIiLSgEWSiIhIg/8HYDIkzXtmzLoAAAAASUVORK5CYII=","text/plain": ["<Figure size 500x300 with 1 Axes>"]@@ -640,7 +584,7 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": 9,"metadata": {"id": "NnARfPRHqZH4"},@@ -655,35 +599,35 @@"Prompt 3 Instruction: Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\n","\n","----------------\n",- "Best program after 0 trials | Score: 35.2:\n",- "Prompt 1 Instruction: Given the fields `context` and `question`, identify the specific information being asked for and produce a concise and accurate response.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment, identify specific works or individuals and provide a concise answer directly corresponding to the question posed.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment, and a list of passages, rank the passages by their relevance to the question. Return a comma-separated list of the numbers associated with each passage, ordered from the most relevant (helpful) to the least relevant.\n",+ "Best program after 0 trials | Score: 43.8:\n",+ "Prompt 1 Instruction: Given a set of context and question pairs, prompt the model to generate a search query that accurately captures the key information needed to answer the question within the given context.\n",+ "Prompt 2 Instruction: Given the provided context about the album \"Plastic Beach\" and the specific question, \"Who released the album that 'Stylo' was a single on in the United States?\", generate the accurate and precise answer to the question.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n",- "Best program after 5 trials | Score: 35.6:\n",- "Prompt 1 Instruction: Given a fact-based question related to pop culture, history, or entertainment, with a focus on identifying specific works or individuals, generate a search query that includes the key elements of the question for precise information retrieval.\n",- "Prompt 2 Instruction: Given the information in the `context`, answer the `question` with a concise and specific response.\n",- "Prompt 3 Instruction: Given a question we are trying to answer and a list of passages, return a comma-separated list of the numbers associated with each passage, ordered by helpfulness in answering the question, with the most helpful passage number first and the least helpful last.\n",+ "Best program after 5 trials | Score: 43.8:\n",+ "Prompt 1 Instruction: Given a set of context and question pairs, prompt the model to generate a search query that accurately captures the key information needed to answer the question within the given context.\n",+ "Prompt 2 Instruction: Given the provided context about the album \"Plastic Beach\" and the specific question, \"Who released the album that 'Stylo' was a single on in the United States?\", generate the accurate and precise answer to the question.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n",- "Best program after 10 trials | Score: 38.2:\n",- "Prompt 1 Instruction: Given the varied syntax and focus on identifying specific works or individuals in fact-based questions related to pop culture, history, and entertainment, generate a search query that retrieves precise information relevant to the specific and varied questions posed.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment and the corresponding context, provide a concise and accurate answer identifying specific works or individuals.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment, and a list of relevant passages, rank the passages by their relevance to the question. Return a comma-separated list of the numbers associated with each passage in order of relevance, with the most helpful passage number first, and the least helpful last.\n",+ "Best program after 10 trials | Score: 43.8:\n",+ "Prompt 1 Instruction: Given a set of context and question pairs, prompt the model to generate a search query that accurately captures the key information needed to answer the question within the given context.\n",+ "Prompt 2 Instruction: Given the provided context about the album \"Plastic Beach\" and the specific question, \"Who released the album that 'Stylo' was a single on in the United States?\", generate the accurate and precise answer to the question.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n",- "Best program after 15 trials | Score: 38.2:\n",- "Prompt 1 Instruction: Given the varied syntax and focus on identifying specific works or individuals in fact-based questions related to pop culture, history, and entertainment, generate a search query that retrieves precise information relevant to the specific and varied questions posed.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment and the corresponding context, provide a concise and accurate answer identifying specific works or individuals.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment, and a list of relevant passages, rank the passages by their relevance to the question. Return a comma-separated list of the numbers associated with each passage in order of relevance, with the most helpful passage number first, and the least helpful last.\n",+ "Best program after 15 trials | Score: 43.8:\n",+ "Prompt 1 Instruction: Given a set of context and question pairs, prompt the model to generate a search query that accurately captures the key information needed to answer the question within the given context.\n",+ "Prompt 2 Instruction: Given the provided context about the album \"Plastic Beach\" and the specific question, \"Who released the album that 'Stylo' was a single on in the United States?\", generate the accurate and precise answer to the question.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n",- "Best program after 20 trials | Score: 41.6:\n",- "Prompt 1 Instruction: Given the fields `context` and `question`, determine the specific information being asked and compile a concise search query to retrieve the requested information.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment, identify and provide the specific individual or work being referred to in the context.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment, and a list of passages, rank the passages in order of relevance to the question. Return a comma-separated list of the corresponding passage numbers, with the most relevant passage first and the least relevant last.\n",+ "Best program after 20 trials | Score: 46.4:\n",+ "Prompt 1 Instruction: Given the provided context and question, generate a search query that will yield the specific information needed to answer the question accurately and directly.\n",+ "Prompt 2 Instruction: Given the provided context about various film directors and their nominations for the Academy Award Oscar, and the specific question asking which director received more nominations, please generate the correct answer based on the information provided.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n",- "Best program after 25 trials | Score: 41.6:\n",- "Prompt 1 Instruction: Given the fields `context` and `question`, determine the specific information being asked and compile a concise search query to retrieve the requested information.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment, identify and provide the specific individual or work being referred to in the context.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment, and a list of passages, rank the passages in order of relevance to the question. Return a comma-separated list of the corresponding passage numbers, with the most relevant passage first and the least relevant last.\n",+ "Best program after 25 trials | Score: 46.4:\n",+ "Prompt 1 Instruction: Given the provided context and question, generate a search query that will yield the specific information needed to answer the question accurately and directly.\n",+ "Prompt 2 Instruction: Given the provided context about various film directors and their nominations for the Academy Award Oscar, and the specific question asking which director received more nominations, please generate the correct answer based on the information provided.\n",+ "Prompt 3 Instruction: Rank the passages in order of helpfulness to answer the question \"Which genus is part of the mallow family; Ferraria or Abelmoschus?\" based on the information provided in the passages. Start by identifying the mentions of the mallow family in each passage and determine if the genus is part of the mallow family. Then, assign a number to each passage in the order of helpfulness, with the most helpful passage having the lowest number and the least helpful passage having the highest number. Separate the numbers with commas.\n","\n"]}@@ -697,7 +641,7 @@" elif (hasattr(predictor, 'signature')):\n"," return predictor.signature\n","\n",- "print(f\"Basline program | Score: {best_score}:\")\n",+ "print(f\"Baseline program | Score: {best_score}:\")\n","for i,predictor in enumerate(program.predictors()):\n"," print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n","print()\n",@@ -707,7 +651,7 @@"for trial_num in compiled_program.trial_logs:\n"," program_score = compiled_program.trial_logs[trial_num][\"score\"]\n"," program_pruned = compiled_program.trial_logs[trial_num][\"pruned\"]\n",- " if program_score > best_score and not program_pruned:\n",+ " if program_score > best_score and not program_pruned and compiled_program.trial_logs[trial_num][\"full_eval\"]:\n"," best_score = program_score\n"," best_program_so_far = compiled_program.trial_logs[trial_num][\"program\"]\n"," if trial_num % 5 == 0:\n",@@ -737,7 +681,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.10.10"+ "version": "3.11.7"},"widgets": {"application/vnd.jupyter.widget-state+json": {