DSPy: The framework for programming—not prompting—language models
Python
merged
klopsahlong5f4cd19b9934
diff --cc dspy/teleprompt/signature_opt_bayesian.pyindex a0cdd8458,bca0d299d..a4091eafa--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@@ -33,9 -38,356 +33,9 @@@ Note that this teleprompter takes in th* pruned: whether or not this program was prunedThis information will be returned as attributes of the best program."""-class BasicGenerateInstruction(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")--class BasicGenerateInstructionWithDataObservations(Signature):- """You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. I will also give you some ``observations`` I have made about the dataset and task. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative."""-- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- observations = dspy.InputField(desc="Observations about the dataset and task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")--class BasicGenerateInstructionWithExamples(dspy.Signature):- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``examples`` of the expected inputs and outputs.--Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative.""")- # attempted_instructions = dspy.InputField(format=str, desc="Previously attempted task instructions, along with their resulting validation score, and an example of the instruction in use on a sample from our dataset.")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- # examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")--class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.--Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")- observations = dspy.InputField(desc="Observations about the dataset and task")- examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")- proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")- proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")--class ObservationSummarizer(dspy.Signature):- ("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")- observations = dspy.InputField(desc="Observations I have made about my dataset")- summary = dspy.OutputField(desc="Two to Three sentence summary of only the most significant highlights of my observations")--class DatasetDescriptor(dspy.Signature):- ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")-- examples = dspy.InputField(desc="Sample data points from the dataset")- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed")--class DatasetDescriptorWithPriorObservations(dspy.Signature):- ("""Given several examples from a dataset please write observations about trends that hold for most or all of the samples. """- """I will also provide you with a few observations I have already made. Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' """- """Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. """- """It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative""")-- examples = dspy.InputField(desc="Sample data points from the dataset")- prior_observations = dspy.InputField(desc="Some prior observations I made about the data")- observations = dspy.OutputField(desc="Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add")--class BayesianSignatureOptimizer(Teleprompter):- def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=False, view_data_batch_size=10):- self.n = n- self.metric = metric- self.init_temperature = init_temperature- self.prompt_model = prompt_model if prompt_model is not None else dspy.settings.lm- self.task_model = task_model if task_model is not None else dspy.settings.lm- self.verbose = verbose- self.track_stats = track_stats- self.teacher_settings = teacher_settings- self.view_data_batch_size = view_data_batch_size-- def _print_full_program(self, program):- for i,predictor in enumerate(program.predictors()):- if self.verbose: print(f"Predictor {i}")- if self.verbose: print(f"i: {self._get_signature(predictor).instructions}")- if self.verbose: print(f"p: {self._get_signature(predictor).fields[-1].name}")- if self.verbose: print("\n")-- def _print_model_history(self, model, n=1):- if self.verbose: print(f"Model ({model}) History:")- model.inspect_history(n=n)-- def _observe_data(self, trainset):- upper_lim = min(len(trainset), self.view_data_batch_size)- observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=(trainset[0:upper_lim].__repr__()))- observations = observation["observations"]-- skips = 0- for b in range(self.view_data_batch_size, len(trainset), self.view_data_batch_size):- upper_lim = min(len(trainset), b+self.view_data_batch_size)- output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=(trainset[b:upper_lim].__repr__()))- if len(output["observations"]) >= 8 and output["observations"][:8].upper() == "COMPLETE":- skips += 1- if skips >= 5:- break- continue- observations += output["observations"]-- summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)-- return summary.summary-- def _create_example_string(self, fields, example):-- # Building the output string- output = []- for field in fields:- name = field.name- separator = field.separator- input_variable = field.input_variable-- # Determine the value from input_data or prediction_data- value = example.get(input_variable)-- # Construct the string for the current field- field_str = f"{name}{separator}{value}"- output.append(field_str)-- # Joining all the field strings- return '\n'.join(output)-- def _get_signature(self, predictor):- if (hasattr(predictor, 'extended_signature')):- return predictor.extended_signature- elif (hasattr(predictor, 'signature')):- return predictor.signature-- def _generate_first_N_candidates(self, module, N, view_data, view_examples, demo_candidates, devset):- candidates = {}- evaluated_candidates = defaultdict(dict)-- if view_data:- # Create data observations- self.observations = None- with dspy.settings.context(lm=self.prompt_model):- self.observations = self._observe_data(devset).replace("Observations:","").replace("Summary:","")-- if view_examples:- example_sets = {}- for predictor in module.predictors():- # Get all augmented examples- example_set = {}- all_sets_of_examples = demo_candidates[id(predictor)] # Get all generated sets of examples- for example_set_i, set_of_examples in enumerate(all_sets_of_examples):- if example_set_i != 0: # Skip the no examples case- for example in set_of_examples: # Get each individual example in the set- if "augmented" in example.keys():- if example["augmented"]:- if example_set_i not in example_set:- example_set[example_set_i] = []- fields_to_use = self._get_signature(predictor).fields- input_variable_names = [field.input_variable for field in fields_to_use]- example_with_only_signature_fields = {key: value for key, value in example.items() if key in input_variable_names}- example_string = self._create_example_string(fields_to_use, example_with_only_signature_fields)- example_set[example_set_i].append(example_string)- example_sets[id(predictor)] = example_set- else:- example_set[example_set_i] = []- example_sets[id(predictor)] = example_set-- # Seed the prompt optimizer zero shot with just the instruction, generate BREADTH new prompts- for predictor in module.predictors():- basic_instruction = None- basic_prefix = None- basic_instruction = self._get_signature(predictor).instructions- basic_prefix = self._get_signature(predictor).fields[-1].name- with dspy.settings.context(lm=self.prompt_model):- # Data & Examples- if view_data and view_examples:- instruct = None- for i in range(1,self.n):- new_instruct = dspy.Predict(BasicGenerateInstructionWithExamplesAndDataObservations, n=1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations, examples=example_sets[id(predictor)][i])- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)- instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)- # Just data- elif view_data:- instruct = dspy.Predict(BasicGenerateInstructionWithDataObservations, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations)- # Just examples- elif view_examples:- instruct = None- for i in range(1,self.n): # Note: skip over the first example set which is empty- new_instruct = dspy.Predict(BasicGenerateInstructionWithExamples, n=1, temperature=self.init_temperature)(basic_instruction=basic_instruction, examples=example_sets[id(predictor)][i])- if not instruct:- instruct = new_instruct- else:- instruct.completions.proposed_instruction.extend(new_instruct.completions.proposed_instruction)- instruct.completions.proposed_prefix_for_output_field.extend(new_instruct.completions.proposed_prefix_for_output_field)- # Neither- else:- instruct = dspy.Predict(BasicGenerateInstruction, n=N-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)-- # Add in our initial prompt as a candidate as well- instruct.completions.proposed_instruction.insert(0, basic_instruction)- instruct.completions.proposed_prefix_for_output_field.insert(0, basic_prefix)- candidates[id(predictor)] = instruct.completions- evaluated_candidates[id(predictor)] = {}-- if self.verbose: self._print_model_history(self.prompt_model)-- return candidates, evaluated_candidates-- def compile(self, student, *, devset, optuna_trials_num, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, view_data=True, view_examples=True):- random.seed(seed)+class BayesianSignatureOptimizer(MIPRO):+ def __init__(self, prompt_model=None, task_model=None, teacher_settings={}, n=10, metric=None, init_temperature=1.0, verbose=False, track_stats=True, view_data_batch_size=10):+ print(u"\u001b[31m[WARNING] BayesianSignatureOptimizer has been deprecated and replaced with MIPRO. BayesianSignatureOptimizer will be removed in a future release. \u001b[31m")-+- # Set up program and evaluation function- module = student.deepcopy()- evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)-- # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt- if max_bootstrapped_demos==0 and max_labeled_demos==0: #TODO: address case when max_bootstrapped alone is 0- max_bootstrapped_demos_for_candidate_gen = 1- max_labeled_demos_for_candidate_gen = 1 #TODO: this might only need to be 0- else:- max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos- max_labeled_demos_for_candidate_gen = max_labeled_demos-- # Generate N few shot example sets- demo_candidates = {}- for i in range(self.n):- if i == 0: # Story empty set of demos as default for index 0- for module_p in module.predictors():- if id(module_p) not in demo_candidates.keys():- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append([])- else:- if self.verbose: print(f"Creating basic bootstrap: {i}/{self.n-1}")-- # Create a new basic bootstrap few - shot program .- rng = random.Random(i)- shuffled_devset = devset[:] # Create a copy of devset- rng.shuffle(shuffled_devset) # Shuffle the copy- tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen, teacher_settings=self.teacher_settings)- candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_devset)-- # Store the candidate demos- for module_p, candidate_p in zip(module.predictors(), candidate_program.predictors()):- if id(module_p) not in demo_candidates.keys():- demo_candidates[id(module_p)] = []- demo_candidates[id(module_p)].append(candidate_p.demos)-- # Generate N candidate prompts- instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, view_data, view_examples, demo_candidates, devset)-- # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples- if max_bootstrapped_demos==0 and max_labeled_demos==0:- demo_candidates = None-- # Initialize variables to store the best program and its score- best_score = float('-inf')- best_program = None- trial_num = 0-- trial_logs = {}-- # Define our trial objective- def create_objective(baseline_program, instruction_candidates, demo_candidates, evaluate, devset):- def objective(trial):- nonlocal best_program, best_score, trial_num, trial_logs # Allow access to the outer variables- candidate_program = baseline_program.deepcopy()-- # Suggest the instruction to use for our predictor- print(f"Starting trial #{trial_num}")- trial_logs[trial_num] = {}-- for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):-- # Get instruction candidates for our given predictor- p_instruction_candidates = instruction_candidates[id(p_old)]- if demo_candidates: p_demo_candidates = demo_candidates[id(p_old)]-- # Suggest the index of the instruction candidate to use in our trial- instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))- if demo_candidates: demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))- trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx- if demo_candidates: trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx-- # Get the selected instruction candidate- selected_candidate = p_instruction_candidates[instruction_idx]- selected_instruction = selected_candidate.proposed_instruction.strip('"').strip()- selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()-- # Use this candidates in our program- self._get_signature(p_new).instructions = selected_instruction- self._get_signature(p_new).fields[-1] = self._get_signature(p_new).fields[-1]._replace(name=selected_prefix)-- # Get the selected demos- if demo_candidates: selected_demos = p_demo_candidates[demos_idx]-- # Use these demos in our program- if demo_candidates: p_new.demos = selected_demos-- if self.verbose: print("Evaling the following program:")- if self.verbose: self._print_full_program(candidate_program)- trial_logs[trial_num]["program"] = candidate_program-- # Evaluate with the new prompts- total_score = 0- batch_size = 100- num_batches = math.ceil(len(devset) / batch_size)-- for i in range(num_batches):- start_index = i * batch_size- end_index = min((i + 1) * batch_size, len(devset))- split_dev = devset[start_index:end_index]- split_score = evaluate(candidate_program, devset=split_dev, display_table=0)- if self.verbose: print(f"{i}st split score: {split_score}")-- total_score += split_score * len(split_dev)- curr_weighted_avg_score = total_score / min((i+1)*100,len(devset))- if self.verbose: print(f"curr average score: {curr_weighted_avg_score}")-- trial.report(curr_weighted_avg_score, i)-- # Handle pruning based on the intermediate value.- if trial.should_prune():- print(f"Trial pruned.")- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = True- trial_num += 1- raise optuna.TrialPruned()-- if self.verbose: print(f"Fully evaled score: {curr_weighted_avg_score}")- if self.verbose: self._print_model_history(self.task_model, n=1)- score = curr_weighted_avg_score-- trial_logs[trial_num]["score"] = curr_weighted_avg_score- trial_logs[trial_num]["pruned"] = False-- # Update the best program if the current score is better- if score > best_score:- best_score = score- best_program = candidate_program.deepcopy()-- trial_num += 1-- return score-- return objective-- # Run the trial- objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, devset)- sampler = optuna.samplers.TPESampler(seed=seed)- study = optuna.create_study(direction="maximize", sampler=sampler)- score = study.optimize(objective_function, n_trials=optuna_trials_num)-- if best_program is not None and self.track_stats:- best_program.trial_logs = trial_logs-- return best_program+ super().__init__(prompt_model, task_model, teacher_settings,n,metric,init_temperature,verbose,track_stats,view_data_batch_size)diff --cc examples/qa/hotpot/hotpotqa_optimized.ipynbindex c87d47497,cf002054e..c9b793a6a--- a/examples/qa/hotpot/hotpotqa_optimized.ipynb+++ b/examples/qa/hotpot/hotpotqa_optimized.ipynb@@@ -57,61 -30,47 +57,32 @@@},{"cell_type": "code",- "execution_count": 4,- "execution_count": 1,++ "execution_count": null,"metadata": {},- "outputs": [- {- "ename": "SyntaxError",- "evalue": "unterminated string literal (detected at line 11) (1838760429.py, line 11)",- "output_type": "error",- "traceback": [- "\u001b[0;36m Cell \u001b[0;32mIn[1], line 11\u001b[0;36m\u001b[0m\n\u001b[0;31m os.environ[\"DSP_CACHEDIR\"] = f\"repo_clone_path/sub_dir\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unterminated string literal (detected at line 11)\n"- ]- }- ],+ "outputs": [],"source": [- "!rm -rf DSPy_optimizer_cache\n",- "!git clone https://huggingface.co/kopsahlong/DSPy_optimizer_cache\n",- "%cd DSPy_optimizer_cache/\n",- "# !git checkout master\n",- "%cd ..\n",+ "from huggingface_hub import hf_hub_download\n",+ "import zipfile\n","import os\n",- "repo_clone_path = 'DSPy_optimizer_cache/cache' #TODO: update this cache to just contain the runs we need!!\n","\n",- "# Set up the cache for this notebook\n",- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path\n",- "os.environ[\"DSP_CACHEDIR\"] = f\"{repo_clone_path}/cachedir\"\n"+ "# Define the repository ID on Hugging Face\n",+ "repo_id = 'kopsahlong/test3'\n",+ "cache_file_path = hf_hub_download(repo_id=repo_id, filename='notebook_cache_v3.zip')\n",+ "compiled_program_file_path = hf_hub_download(repo_id=repo_id, filename='compiled_program.pickle')\n",+ "# Unzipping the file\n",+ "with zipfile.ZipFile(cache_file_path, 'r') as zip_ref:\n",+ " zip_ref.extractall(\".\")\n",+ "\n",- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"notebook_cache\"\n",- "\n",- "\n",- "# Set up the cache for this notebook\n",- "# os.environ[\"DSP_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/caches/cache_train_500_eval_500_n_10_trials_30_hops_4\" # repo_clone_path\n",- "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/caches/MIPRO_notebook_cache_v2\" # repo_clone_path\n",- "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/notebook_cache\" # repo_clone_path\n",- "# os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"/lfs/0/kristaoo/dspy/examples/qa/hotpot/DSPy_notebook_cache/cache_copy\" "- ]- },- {- "cell_type": "code",- "execution_count": 7,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Pydantic version: 2.6.3\n"- ]- }- ],- "source": [- "import pkg_resources\n",- "\n",- "# Get the version of cloudpickle\n",- "pydantic_v = pkg_resources.get_distribution(\"pydantic\").version\n",- "\n",- "print(f\"Pydantic version: {pydantic_v}\")\n"++ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"notebook_cache\""]},{"cell_type": "code",- "execution_count": 11,- "execution_count": 3,++ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "/lfs/0/kristaoo/home\n"- ]- }- ],+ "outputs": [],"source": [- "!echo $HOME"+ "# TODO: add in DSPy setup"]},{@@@ -123,9 -82,18 +94,9 @@@},{"cell_type": "code",- "execution_count": 12,- "execution_count": 2,++ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",- " from .autonotebook import tqdm as notebook_tqdm\n"- ]- }- ],+ "outputs": [],"source": ["import os \n","import dspy\n",@@@ -165,18 -135,18 +136,9 @@@},{"cell_type": "code",- "execution_count": 13,- "execution_count": 3,++ "execution_count": null,"metadata": {},-- "outputs": [-- {-- "name": "stderr",-- "output_type": "stream",-- "text": [-- "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.\n",-- " table = cls._concat_blocks(blocks, axis=0)\n"-- ]-- }-- ],++ "outputs": [],"source": ["from dspy.evaluate import Evaluate\n","import re \n",@@@ -256,41 -226,54 +218,12 @@@},{"cell_type": "code",- "execution_count": 14,- "execution_count": 4,++ "execution_count": null,"metadata": {},-- "outputs": [-- {-- "name": "stderr",-- "output_type": "stream",-- "text": [- "Average Metric: 80 / 500 (16.0): 100%|██████████| 500/500 [00:30<00:00, 16.32it/s]\n"- " 0%| | 0/500 [00:00<?, ?it/s]"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/joblib/memory.py:655: JobLibCollisionWarning: Possible name collisions between functions 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:-1) and 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:52)\n",- " return self._cached_call(args, kwargs)[0]\n",- "Average Metric: 80 / 500 (16.0): 100%|██████████| 500/500 [01:19<00:00, 6.26it/s]\n",- "/lfs/0/kristaoo/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",- " df = df.applymap(truncate_cell)\n"-- ]-- },-- {-- "name": "stdout",-- "output_type": "stream",-- "text": [-- "Average Metric: 80 / 500 (16.0%)\n"-- ]-- },-- {-- "name": "stderr",-- "output_type": "stream",-- "text": [- "Average Metric: 107 / 500 (21.4): 100%|██████████| 500/500 [00:29<00:00, 17.01it/s]\n"- "Average Metric: 107 / 500 (21.4): 100%|██████████| 500/500 [01:20<00:00, 6.24it/s]\n"-- ]-- },-- {-- "name": "stdout",-- "output_type": "stream",-- "text": [-- "Average Metric: 107 / 500 (21.4%)\n"-- ]-- }-- ],++ "outputs": [],"source": ["baseline_train_score = evaluate(program,devset=trainset)\n",- "baseline_eval_score = evaluate(program, devset=devset)\n",- "\n",- "#TODO: add in retrieval eval "+ "baseline_eval_score = evaluate(program, devset=devset)"]},{@@@ -308,70 -291,10 +241,42 @@@]},{- "cell_type": "markdown",+ "cell_type": "code",- "execution_count": 5,++ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "import cloudpickle as pickle\n",+ "from dspy.teleprompt import BayesianSignatureOptimizer\n",+ "\n",+ "LOAD_PRECOMPILED_PROGRAM = True\n",+ "\n",+ "# By default, we will load the precompiled program\n",+ "if LOAD_PRECOMPILED_PROGRAM:\n",+ " # Load a our precompiled program\n",+ " with open(compiled_program_file_path, 'rb') as file:\n",+ " # Load the data from the file\n",+ " compiled_program = pickle.load(file)\n",+ "# Otherwise, if desired, the program can be compiled from scratch \n",+ "else:\n",+ " # Define hyperparameters:\n",+ " N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",+ " trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",+ " temperature = 1.0 # The temperature configured for generating new instructions\n",+ "\n",+ " # Compile\n",+ " eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",+ " teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=True)\n",+ " compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"+ ]+ },+ {+ "cell_type": "code",- "execution_count": 6,++ "execution_count": null,"metadata": {},- "outputs": [- {- "data": {- "text/plain": [- "generate_query = ChainOfThought(StringSignature(context, question -> search_query\n",- " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",- "))\n",- "generate_answer = ChainOfThought(StringSignature(context, question -> answer\n",- " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",- " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",- " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",- " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",- "))\n",- "generate_ranking = ChainOfThought(ReturnRankedDocuments(question, context -> ranking\n",- " instructions='Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.'\n",- " question = Field(annotation=str required=True json_schema_extra={'desc': \"The question we're trying to answer.\", '__dspy_field_type': 'input', 'prefix': 'Question:'})\n",- " context = Field(annotation=str required=True json_schema_extra={'desc': 'List of potentially related passages.', '__dspy_field_type': 'input', 'prefix': 'Context:'})\n",- " ranking = Field(annotation=str required=True json_schema_extra={'desc': 'A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.', '__dspy_field_type': 'output', 'prefix': 'Ranking:'})\n",- "))"- ]- },- "execution_count": 6,- "metadata": {},- "output_type": "execute_result"- }- ],++ "outputs": [],"source": [- "#### 3a] Inspecting Pre-Optimized Program"+ "compiled_program"]},{@@@ -383,21 -306,10 +288,13 @@@},{"cell_type": "code",- "execution_count": 16,+ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 84 / 201 (41.8): 40%|████ | 200/500 [00:13<00:19, 15.48it/s]"- ]- }- ],+ "outputs": [],- "source": []+ "source": [+ "bayesian_train_score = evaluate(compiled_program, devset=trainset)\n",+ "bayesian_eval_score = evaluate(compiled_program, devset=devset)"+ ]},{"cell_type": "markdown",@@@ -415,46 -327,1521 +312,35 @@@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": null,"metadata": {},-- "outputs": [-- {- "data": {- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAE8CAYAAACrYErbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABP3UlEQVR4nO3deVxUZfs/8M8MssOAKJvCACoCKmjgEiUIioILoWBk+JS4ZkFBpKX1mGiLS08FmsvzlKlZmEtom6KIoqBoSq45kRI0pqBGyYAoDDP37w9/c76OwwAzDMzAXO/Xi9fLOeeeM9c158jFuc99n8NjjDEQQgghRAVf3wEQQgghhoqKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJGlWWFgYwsLCtHovj8dDenq6TuMxZpp8nzweD8nJye0bUAfbsmULeDweysvL9R0KMSJUJLs4Ho/Xqp/8/Hy9xXj79m2kpKTA19cXlpaWcHJywvDhw/HGG2+gtrZWb3EZuhMnTiA9PR137tzR6XbLy8uVjg0TExMIhUJMmTIF586d0+lndSaFhYUYP348evfuDQsLCwiFQkRHRyMrK4trU1dXh/T09Db9f2qv/Uq0003fAZD2tW3bNqXXX3zxBXJzc1WW+/n5Nfn+gwcPtltsAPD3339j6NChkEgkmDVrFnx9fVFVVYULFy5gw4YNePHFF2FjY9OuMXQW9+7dQ7du//df9sSJE1i2bBkSExNhb2+v88979tlnMWHCBMhkMohEImzYsAH79+/HyZMnMWTIEJ1/Xkuee+45TJs2Debm5h3+2bt27cIzzzyDIUOGICUlBd27d0dZWRmOHTuGTz/9FAkJCQAeFMlly5YBgNY9MO29X4lmqEh2cf/617+UXp88eRK5ubkqyx9VV1cHKysrmJmZtWd42LRpE8RiMY4fP44nnnhCaZ1EImn3z3/Y3bt3YW1t3WGfpykLC4sO/bzAwECl4+TJJ5/EU089hQ0bNuC///1vk+9pz+/QxMQEJiYm7bLtlqSnp2PAgAE4efKkyjF569YtvcREOgZ1txKEhYVh0KBBKC4uRmhoKKysrPDmm29y6x7+i7ihoQFvv/02goKCYGdnB2tra4SEhODIkSNafXZpaSlMTEzw+OOPq6wTCAQqheHUqVOYMGECunfvDmtrawQEBCAzM1OpzeHDhxESEgJra2vY29sjJiYGIpFIqU16ejp4PB4uX76MhIQEdO/eHSNHjuTWf/nllwgKCoKlpSUcHBwwbdo0XLt2TWkbV65cQVxcHFxcXGBhYQE3NzdMmzYN1dXVavNds2YNTExMlLrSPvzwQ/B4PKSlpXHLZDIZbG1t8cYbb3DLHr4mmZ6ejoULFwIAvLy8uK7RR6/X7d27F4MGDYK5uTkGDhyInJwctbG1ZPTo0QCAsrIyAP93jfDo0aN46aWX4OTkBDc3NwBAYmIiPD09Vbah+N4fprh+2lKsTV2T9PT0xKRJk1BYWIjhw4fDwsICffr0wRdffKHy2RcuXMCoUaNgaWkJNzc3vPvuu9i8eXOrrnOWlpZi2LBhTf7R5uTkBOBBN7WjoyMAYNmyZdw+UeyzCxcuIDExEX369IGFhQVcXFwwa9YsVFVVKX0/6varoht8y5YtKjE8er26pqYGqamp8PT0hLm5OZycnDB27Fj8/PPPzeZJVNGZJAEAVFVVYfz48Zg2bRr+9a9/wdnZucl2EokEn332GZ599lnMnTsXNTU12LRpEyIjI/HTTz9p3A3n4eEBmUyGbdu2YcaMGc22zc3NxaRJk+Dq6oqUlBS4uLhAJBLhhx9+QEpKCgDg0KFDGD9+PPr06YP09HTcu3cPa9euxZNPPomff/5Z5Rf3008/DW9vb7z//vtQPDXuvffew5IlSxAfH485c+bg9u3bWLt2LUJDQ3H27FnY29ujoaEBkZGRqK+vx8svvwwXFxdcv34dP/zwA+7cuQM7O7smcwgJCYFcLkdhYSEmTZoEACgoKACfz0dBQQHX7uzZs6itrUVoaGiT24mNjcVvv/2G7du34+OPP0bPnj0BgPslDTy4hpadnY2XXnoJtra2WLNmDeLi4iAWi9GjR49mv+umlJaWAoDKe1966SU4Ojri7bffxt27dzXebltjvXr1KqZOnYrZs2djxowZ+Pzzz5GYmIigoCAMHDgQAHD9+nWEh4eDx+Nh8eLFsLa2xmeffdbqrlsPDw/k5eXhzz//5P4QeJSjoyN3iWDKlCmIjY0FAAQEBAB4cPz+/vvvmDlzJlxcXPDLL7/gf//7H3755RecPHkSPB6v2f16+/btVsUKAPPnz8fu3buRnJyMAQMGoKqqCoWFhRCJRAgMDGz1dggARoxKUlISe3S3jxo1igFgGzduVGk/atQoNmrUKO51Y2Mjq6+vV2rzzz//MGdnZzZr1iyl5QDY0qVLm42nsrKSOTo6MgDM19eXzZ8/n2VlZbE7d+4otWtsbGReXl7Mw8OD/fPPP0rr5HI59+8hQ4YwJycnVlVVxS07f/484/P57Pnnn+eWLV26lAFgzz77rNK2ysvLmYmJCXvvvfeUll+8eJF169aNW3727FkGgO3atavZ/B4lk8mYQCBgr7/+Ohd7jx492NNPP81MTExYTU0NY4yxjz76iPH5fKVcH/0+P/jgAwaAlZWVqXwOAGZmZsauXr2q9D0AYGvXrm02xrKyMgaALVu2jN2+fZtVVlay/Px89thjjzEA7JtvvmGMMbZ582YGgI0cOZI1NjYqbWPGjBnMw8NDZduK712bWBWf93C+Hh4eDAA7duwYt+zWrVvM3Nycvfbaa9yyl19+mfF4PHb27FluWVVVFXNwcFD7HT5s06ZNXJzh4eFsyZIlrKCggMlkMqV2t2/fVnvc19XVqSzbvn27Svzq9qtiv2zevFllO49+pp2dHUtKSmo2J9I61N1KAADm5uaYOXNmi+1MTEy4Lie5XI6///4bjY2NGDp0qFZdOc7Ozjh//jzmz5+Pf/75Bxs3bkRCQgKcnJzwzjvvcGd3Z8+eRVlZGVJTU1UGMyi67yoqKnDu3DkkJibCwcGBWx8QEICxY8di3759Kp8/f/58pdfZ2dmQy+WIj4/HX3/9xf24uLjA29ub61ZWnCkeOHAAdXV1rc6Xz+fjiSeewLFjxwAAIpEIVVVVWLRoERhjKCoqAvDg7HLQoEFtGrgRERGBvn37cq8DAgIgEAjw+++/t+r9S5cuhaOjI1xcXBAWFobS0lKsWrWKO0NSmDt3bpuvFbYl1gEDBiAkJIR77ejoCB8fH6X35uTkIDg4WKmnw8HBAdOnT29VfLNmzUJOTg7CwsJQWFiId955ByEhIfD29saJEydatQ1LS0vu3/fv38dff/3FXWbQdTeovb09Tp06hRs3buh0u8aIiiQBAPTu3bvVg2S2bt2KgIAAWFhYoEePHnB0dMSPP/7Y7LW45ri6umLDhg2oqKhASUkJ1qxZw3Xfbdq0CcD/dfUNGjRI7Xb++OMPAICPj4/KOj8/P/z1118q3YFeXl5Kr69cuQLGGLy9veHo6Kj0IxKJuEEaXl5eSEtLw2effYaePXsiMjIS69ata9V3EBISguLiYty7dw8FBQVwdXVFYGAgBg8ezHW5FhYWKv3i14ZQKFRZ1r17d/zzzz+tev+8efOQm5uLvLw8FBcX49atW3j99ddV2j36HXZ0rK157x9//IF+/fqptGtqmTqRkZE4cOAA7ty5g2PHjiEpKQl//PEHJk2a1KrBO3///TdSUlLg7OwMS0tLODo6ct+dtv931Fm9ejUuXboEd3d3DB8+HOnp6a3+44goo2uSBIDyX7nN+fLLL5GYmIjJkydj4cKFcHJygomJCVasWMEVMm3xeDz0798f/fv3x8SJE+Ht7Y2vvvoKc+bMadN2m/No3nK5HDweD/v372/y7Ojh6SgffvghEhMT8e233+LgwYN45ZVXsGLFCpw8eVLtdSsAGDlyJKRSKYqKilBQUMAVw5CQEBQUFODXX3/F7du321wk1Z3dKc7OW+Lt7Y2IiIgW2zV17Dw6OEdBJpM1ubwtsbY1T01ZWVkhJCQEISEh6NmzJ5YtW4b9+/e3eE09Pj4eJ06cwMKFCzFkyBDY2NhALpcjKioKcrm8xc/V5DuNj49HSEgI9uzZg4MHD+KDDz7AqlWrkJ2djfHjx7cuUQKAiiTR0O7du9GnTx9kZ2cr/addunSpTj+nT58+6N69OyoqKgCA64q7dOmS2l/cHh4eAICSkhKVdb/++it69uzZ4vSEvn37gjEGLy8v9O/fv8U4/f394e/vj3//+984ceIEnnzySWzcuBHvvvuu2vcMHz4cZmZmKCgoQEFBATeaMTQ0FJ9++iny8vK4181R90vTEHTv3r3JyfCKs/2O5uHhgatXr6osb2qZJoYOHQoA3HGqbp/8888/yMvLw7Jly/D2229zy69cuaLSVt02unfvDgAq36u679TV1RUvvfQSXnrpJdy6dQuBgYF47733qEhqiLpbiUYUf7U//Ff6qVOnuGtpmjp16lSTIyJ/+uknVFVVcV2ngYGB8PLyQkZGhsovCUUsrq6uGDJkCLZu3arU5tKlSzh48CAmTJjQYjyxsbEwMTHBsmXLVM5EGGPccH2JRILGxkal9f7+/uDz+aivr2/2MywsLDBs2DBs374dYrFY6Uzy3r17WLNmDfr27QtXV9dmt6Mo+IZ4Z5a+ffuiuroaFy5c4JZVVFRgz549eoknMjISRUVFSncM+vvvv/HVV1+16v2KP1wepbjOrThOraysAKjuk6b+3wBARkaGyjbV7VeBQICePXty17MV1q9fr/RaJpOpdN86OTmhV69eLR6bRBWdSRKNTJo0CdnZ2ZgyZQomTpyIsrIybNy4EQMGDNDqFnLbtm3DV199hSlTpiAoKAhmZmYQiUT4/PPPYWFhwc3X5PP52LBhA6KjozFkyBDMnDkTrq6u+PXXX/HLL7/gwIEDAIAPPvgA48ePR3BwMGbPns1NAbGzs2vVfU/79u2Ld999F4sXL0Z5eTkmT54MW1tblJWVYc+ePZg3bx4WLFiAw4cPIzk5GU8//TT69++PxsZGbNu2DSYmJoiLi2vxc0JCQrBy5UrY2dnB398fwINfZD4+PigpKUFiYmKL2wgKCgIAvPXWW5g2bRpMTU0RHR1tEDdEmDZtGt544w1MmTIFr7zyCurq6rBhwwb0799fL3P1Xn/9dXz55ZcYO3YsXn75ZW4KiFAoxN9//93iWXlMTAy8vLwQHR2Nvn374u7duzh06BC+//57DBs2DNHR0QAedD0PGDAAO3bsQP/+/eHg4IBBgwZh0KBBCA0NxerVqyGVStG7d28cPHiQm3P6sOb265w5c7By5UrMmTMHQ4cOxbFjx/Dbb78pvb+mpgZubm6YOnUqBg8eDBsbGxw6dAinT5/Ghx9+qKNv1IjoZ1At0Rd1U0AGDhzYZPtHp4DI5XL2/vvvMw8PD2Zubs4ee+wx9sMPPzQ55B+tmAJy4cIFtnDhQhYYGMgcHBxYt27dmKurK3v66afZzz//rNK+sLCQjR07ltna2jJra2sWEBCgMqXh0KFD7Mknn2SWlpZMIBCw6OhodvnyZaU2iqkIt2/fbjKub775ho0cOZJZW1sza2tr5uvry5KSklhJSQljjLHff/+dzZo1i/Xt25dZWFgwBwcHFh4ezg4dOtRsvgo//vgjA8DGjx+vtHzOnDkMANu0aZPKe5r6Pt955x3Wu3dvxufzlaYNAGhyCoCHhwebMWNGs7Epphp88MEHzbZTTMk4ffp0k+sPHjzIBg0axMzMzJiPjw/78ssv1U4BaU2s6qaATJw4UeW9jx63jD2YthMSEsLMzc2Zm5sbW7FiBVuzZg0DwCorK5vNdfv27WzatGmsb9++zNLSkllYWLABAwawt956i0kkEqW2J06cYEFBQczMzExpn/35559sypQpzN7entnZ2bGnn36a3bhxQ6P9WldXx2bPns3s7OyYra0ti4+PZ7du3VLaRn19PVu4cCEbPHgw9/9k8ODBbP369c3mSJrGY6ydrm4TQoiBS01NxX//+1/U1tbq7ZZ3xLDRNUlCiFG4d++e0uuqqips27YNI0eOpAJJ1KJrkoQQoxAcHIywsDD4+fnh5s2b2LRpEyQSCZYsWaLv0IgBoyJJCDEKEyZMwO7du/G///0PPB4PgYGB2LRpU4tTbYhxo2uShBBCiBp0TZIQQghRg4okIYQQokaXvyYpl8tx48YN2NraGvRtvAghhLQfxhhqamrQq1cv8PmtPz/s8kXyxo0bcHd313cYhBBCDMC1a9eafQDBo7p8kbS1tQXw4IsRCARab0cqleLgwYMYN24cTE1NdRWewTK2fAHK2RhyNrZ8AcpZkbNEIoG7uztXE1qryxdJRRerQCBoc5G0srKCQCAwigPN2PIFKGdjyNnY8gUo50dz1vSyGw3cIYQQQtSgIkkIIYSoQUWSEEK6KLlcDrFYDAAQi8WQy+V6jqjz6fLXJAkhxBiJRCLk5OSgtrYWAQEByMrKgo2NDaKiouDn56fv8DoNOpMkhJAuRiQSYefOnZBIJErLJRIJdu7cCZFIpKfIOh8qkoQQ0oXI5XLk5OQ02yYnJ4e6XluJiiQhhHQhYrFY5QzyURKJhLtWSZpHRZIQQrqQmpoanbYzdlQkCSGkC2ntHWU0vfOMsaIiSQghXYhQKGzx7mICgQBCobCDIurcqEgSQkgXwufzERUV1WybqKgojZ6EYczoWyKEkC7Gz88P8fHxKmeUAoEA8fHxNE9SA3QzAUII6YL8/Pzg4+ODsrIyXLp0CQkJCfDy8qIzSA3Rt0UIIV0Un8/nrj0KhUIqkFqgM0lCDJBMLkOBuAAVNRVwtXVFiDAEJnwTfYdFiNGhIkmIgckWZSMlJwV/Sv7klrkJ3JAZlYlYv1g9RkaI8aFzb0IMSLYoG1N3TlUqkABwXXIdU3dORbYoW0+REaIbcrkc5eXluHjxIsrLyw3+9nh0JkmIgZDJZUjJSQEDU1nHwMADD6k5qYjxiaGuV9IpKZ5M8vBt8wQCgUE/mYTOJAkxEAXiApUzyIcxMFyTXEOBuKADoyJENzrrk0noTJJ0SZoOfJHJZSgUFwIACsWFCPUKbbG9pttvqX1FTUWrcmttO10zxsFExphze2jtk0l8fHwMbgQuFUnS5Wg68EXRvqq2CtsDtmNi1kT0sOnRYntNt99Se1db11bl19p2umSMg4mMMef2osmTSTw9PTsmqFYyrJJNSBtpOvDFkNqHCEPgJnADD7wmc+OBB3eBO0KEIc18A7pnjIOJtM1ZJpchvzwf2y9uR355PmRyWUeEa/A685NJqEiSLqOlgS8AkJqTyv3iMrT2JnwTZEZlAoBKoVS8zojK6NDuPk1z6Aq0zTlblA3PTE+Ebw1HQnYCwreGwzPTs9k/IoylqHbmJ5NQkSRdhqYDXwytPQDE+sVid/xu9Bb0VmrrJnDD7vjdHd7NZ4yDibTJWZszT22KamfVmZ9MQkWSdBmaDnwxtPYKsX6xKE8px5EZR5AVm4UjM46gLKVML9fBDH0wUXvQNGdtzjyNrQu7Mz+ZxPAiIkRLmg58MbT2DzPhmyDMMwzP+j+LMM8wvY2oNOTBRO1F05w1PfM0xi5soPM+mYRGt5IuQzHw5brkepO/gHjgwU3gxg18MbT2hqgr5KApTXPW9MxTk6Ia5hmmeQIGTPFkErFYjJqaGtja2hr8jdcNJrKVK1eCx+MhNTWVW3b//n0kJSWhR48esLGxQVxcHG7evKm/IIlB03Tgi6G1N0RdIQdNaZqzpmeextiF/TA+nw9PT0/4+/vD09PToAskYCBF8vTp0/jvf/+LgIAApeWvvvoqvv/+e+zatQtHjx7FjRs3EBtL85OIepoOfDG09oaoK+SgKU1y1nTqjjF2YXdmeu9ura2txfTp0/Hpp5/i3Xff5ZZXV1dj06ZNyMrKwujRowEAmzdvhp+fH06ePInHH39cXyETAxfrF4sYn5hW3ylF0f5Y2TFILknwY8KPzd5xR9vtd+Y7t3SFHDTV2pwVZ55Td04FDzylLtqmzjyNsQu7M9N7kUxKSsLEiRMRERGhVCSLi4shlUoRERHBLfP19YVQKERRUZHaIllfX4/6+nruteIuD1KpFFKpVOs4Fe9tyzY6k66Q75O9n+T+LZfJIZc1/7SBEa4jkHspFyNcR7Sqvabb17R9R9B0P7dnDnK5HH/++Sdqa2thY2MDNzc3nXfFaXNctybn6H7R2B23G28cegPXa65zy91s3bAyYiWi+0UrfWbm2Ew8t+c5AGi6qI7N0Nn32xX+L2uqqZy1zZ/HGFP9U6aDfP3113jvvfdw+vRpWFhYICwsDEOGDEFGRgaysrIwc+ZMpYIHAMOHD0d4eDhWrVrV5DbT09OxbNkyleVZWVmwsrJqlzwIIYQYtrq6OiQkJKC6urrFOZsP09uZ5LVr15CSkoLc3FxYWFjobLuLFy9GWloa91oikcDd3R3jxo3T6It5lFQqRW5uLsaOHQtTU1NdhGrQjC1fgHLWV84lJSXYs2eP2vVTpkyBj4+PTj7LEPJ9mEwuQ9GfRaisrYSLjQuC3YJ13oVtaDl3hKZybuneserorUgWFxfj1q1bCAwM5JbJZDIcO3YMn3zyCQ4cOICGhgbcuXMH9vb2XJubN2/CxcVF7XbNzc1hbm6ustzU1FQnB4iuttNZGFu+AOXckeRyOXJzc5t98G5ubi4GDBig065XQ9nHpjBFeN/wjvksA8m5Iz2cs7a5621065gxY3Dx4kWcO3eO+xk6dCimT5/O/dvU1BR5eXnce0pKSiAWixEcHKyvsAkhOqTJ0yEI0Qe9nUna2tpi0KBBSsusra3Ro0cPbvns2bORlpYGBwcHCAQCvPzyywgODqaRrYR0EZ356RDEOOh9dGtzPv74Y/D5fMTFxaG+vh6RkZFYv369vsMihOhIZ346BDEOBlUk8/PzlV5bWFhg3bp1WLdunX4CIoS0K8XTIZrrcjXUp0MQ42AQd9whhBinzvx0CGIc6MgjhOhVZ306BDEOBtXdSggxTp3x6RD6IJfL6TvqYFQkSacgk8uM6r6hxkjxdAjSNJFIhJycHKXrtwKBAFFRUXS23Y6oSBKDly3KRkpOitIz+NwEbsiMyuyST6Ag5FEikQg7d+5UWS6RSLBz507qlm5HdJ5ODFq2KBtTd05VeUjtdcl1TN05FdmibD1FRkjHkMvlyMnJabZNTk5Os3ctItqjIkkMlkwuQ0pOSpOPE1IsS81JhUwu6+jQCOkwdFci/aIiSQxWgbhA5QzyYQwM1yTXUCAu6MCoCOlYdFci/aIiSQxWRU2FTtsR0hnRXYn0i4okMViutq46bUeUyeQyFIoLAQCF4kLqtjZQirsSNYfuStR+qEgSgxUiDIGbwI17WvujeODBXeCOEGFIB0fW+WWLsuGZ6YmJWRMBABOzJsIz05MGQhkguiuRftG3SgyWCd8EmVGZAKBSKBWvM6IyaL7k/yeXy1FeXo6LFy+ivLxc7WhHYx0xrJiIDzwYDNOZRoPSXYn0h+ZJEoMW6xeL3fG7m5wnmRGVQfMk/7/WTjRvacQwDzyk5qQixiemS/3xofh+amtrERAQgKysLNjY2HSqifh0VyL9oCJJDF6sXyxifGLojjtqaDLRXJMRw2GeYe0Vcod6+Pt5uKB0xon4dFeijkdFknQKJnyTLvNLW5daO9Hcx8cHfD7f6EYMa/r9EPIoOioI6cQ0nWhubCOGaSI+aSsqkoR0YppONDe2EcM0EZ+0FRVJQjoxTSeaG9uIYZqIT9qKiiQhnZg2E80VI4Z7C3ortXMTuGF3/O4uNWKYJuKTtqIiSUgnpu1E81i/WJSnlOPHhB8BAD8m/IiylLIuVSABmohP2o6ODEI6OW0nmpvwTTBSOBIAMFI4sst0sT6KJuKTtqApIIR0ATTRvHmK76esrAyXLl1CQkICvLy86PshLaIiSUgXQRPNm8fn8yEUCnHp0iX6A4K0Gh0lhBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGzZM0ADK5zOgeKGyMORNCOh8qknqWLcpGSk6K0tPi3QRuyIzKVHsfzc5eYLTJmRBC9IG6W/UoW5SNqTunKhULALguuY6pO6ciW5Td5Hs8Mz0RvjUcCdkJCN8aDs9MzybbGiJtciaEEH2hIqknMrkMKTkpYGAq6xTLUnNSIZPLuOWdvcBokzMhhOgTFUk9KRAXqBS7hzEwXJNcQ4G4AEDXKDCa5kwIIfqm1yK5YcMGBAQEQCAQQCAQIDg4GPv37+fW379/H0lJSejRowdsbGwQFxeHmzdv6jHi1pHJZcgvz8f2i9uRX57fZOGqqKlo1bYU7bpCgdE0Z0II0Te9Dtxxc3PDypUr4e3tDcYYtm7dipiYGJw9exYDBw7Eq6++ih9//BG7du2CnZ0dkpOTERsbi+PHj+sz7Ga1dlCKq61rq7anaNcVCoymORNCiL7p9UwyOjoaEyZMgLe3N/r374/33nsPNjY2OHnyJKqrq7Fp0yZ89NFHGD16NIKCgrB582acOHECJ0+e1GfYamlyzTBEGAI3gRt44DW5LR54cBe4I0QYAqBrFBhNcyaEEH0zmCkgMpkMu3btwt27dxEcHIzi4mJIpVJERERwbXx9fSEUClFUVITHH3+8ye3U19ejvr6eey2RSAAAUqkUUqlU6/gU71W3DZlchjcOvAELvkWT63ngYdGBRZjQZwI3XSNzbCae2/McAChda1QUkYyxGZDL5JDL5Hjc9XH0s+uHGzU3mrwuyQMPvW1743HXx9uUp0JL+WpLk5w7WnvlbMiMLWdjyxegnB9dpikeY0z1N24HunjxIoKDg3H//n3Y2NggKysLEyZMQFZWFmbOnKlU8ABg+PDhCA8Px6pVq5rcXnp6OpYtW6ayPCsrC1ZWVu2SAyGEEMNWV1eHhIQEVFdXQyAQtPp9ej+T9PHxwblz51BdXY3du3djxowZOHr0qNbbW7x4MdLS0rjXEokE7u7uGDdunEZfzKOkUilyc3MxduxYmJqaqqzffXk3Zn83u8XtbHpqE6YOmKq0TCaXoejPIlTWVsLFxgXBbsFqbw7wfcn3eOPQG7hec51b5mbrhpURKxHtE61hVqpKSkpw6NAh3L17F4MGDcKlS5dgbW2NiIgI+Pj4tHn7Cprk3FFa2sddUWfNWXGc1tTUcMtsbW1bPE47a75tQTk/yFnRq6gpvRdJMzMz9OvXDwAQFBSE06dPIzMzE8888wwaGhpw584d2Nvbc+1v3rwJFxcXtdszNzeHubm5ynJTU1OdHCDqtuNq54p78nstvt/VzlXl/aYwRXjf8FZ9fuygWMQMiGmXO+6IRCJ88803AAA+/8HlarlcjurqanzzzTeIj4+Hn59fmz8H0CznjqarY6Uz6Uw5P3ycPkyT47Qz5asrxp6ztrkb3DxJuVyO+vp6BAUFwdTUFHl5edy6kpISiMViBAcH6zHCpnXkoBQTvgnCPMPwrP+zCPMM00mBlMvlyMnJabZNTk4O5PKOv1ZIiAIdp6Sj6bVILl68GMeOHUN5eTkuXryIxYsXIz8/H9OnT4ednR1mz56NtLQ0HDlyBMXFxZg5cyaCg4PVDtrRJxO+CTKjMgFApVByg1KiMvTepaiOWCxusTtCIpFALBZ3UESEqKLjlHQ0vXa33rp1C88//zwqKipgZ2eHgIAAHDhwAGPHjgUAfPzxx+Dz+YiLi0N9fT0iIyOxfv16fYbcrFi/WOyO393kPMmMqAyDvnn3w9d2dNGOELlcDrFYjJqaGtja2kIoFHLd+Nqi49T4tMdxpAm9FslNmzY1u97CwgLr1q3DunXrOiiitov1i0WMT/tcM2xPtra2Om1HjJtIJEJOTo7SWZ9AIEBUVFSbrmvTcWpc2us40oTBXZPsCtrjmmF7EwqFLY7+FQgEEAqFHRQR6axEIhF27typ0i0qkUiwc+dOiEQirbdNx6nxaM/jSBNUJAmAB6NZo6Kimm0TFRXVod0cpPNp74E1dJwaB0MaoEVHEuH4+fkhPj5e5S91gUCg0+kfpOvqiIE1dJx2fYY0QEvv8ySJYfHz84OPjw/Kyspw6dIlJCQkwMvLi/4yJ63SUQNrFMepPgd0kPZjSAO0qEgSFXw+H0KhEJcuXaJfPEQjHTmwhs/nw9PTs83bIYbHkAZo0W8/QojO0MAaoguGdBxRkSSE6AwNrCG6YEjHER2phBCdooE1RBcM5Tiia5KEEJ2jgTVEFwzhOKIiSQhpFzSwhuiCvo8j+rOOEEIIUaNNRbKhoQElJSVobGzUVTyEEEKIwdCqSNbV1WH27NmwsrLCwIEDubsevPzyy1i5cqVOAySEEEL0RasiuXjxYpw/fx75+fmwsLDglkdERGDHjh06C44QQgjRJ60G7uzduxc7duzA448/Dh7v/x4wPHDgQJSWluosOEIIIUSftDqTvH37NpycnFSW3717V6loEkIIIZ2ZVkVy6NCh+PHHH7nXisL42WefITg4WDeRGRCZXIZCcSEAoFBcCJlcpueICCGEdAStulvff/99jB8/HpcvX0ZjYyMyMzNx+fJlnDhxAkePHtV1jHqVLcpGSk4KqmqrsD1gOyZmTUQPmx7IjMpErF+svsMjhBDSjrQ6kxw5ciTOnz+PxsZG+Pv74+DBg3ByckJRURGCgoJ0HaPeZIuyMXXnVPwp+VNp+XXJdUzdORXZomw9RUYIIaQjaHwmKZVK8cILL2DJkiX49NNP2yMmgyCTy5CSkwIGprKOgYEHHlJzUhHjEwMTvokeIiSEENLeND6TNDU1xTfffNMesRiUAnGByhnkwxgYrkmuoUBc0IFREUII6UhadbdOnjwZe/fu1XEohqWipkKn7QghhHQ+Wg3c8fb2xvLly3H8+HEEBQXB2tpaaf0rr7yik+D0ydXWVaft9Ekml6FAXICKmgq42roiRBhCXcSEENIKWhXJTZs2wd7eHsXFxSguLlZax+PxukSRDBGGwE3ghuuS601el+SBBzeBG0KEIXqIrvUUo3Mf7jp2E7jR6FxCCGkFrYpkWVmZruMwOCZ8E2RGZWLqzqngQfkGCYrXGVEZBn1Gphid+2iRV4zO3R2/mwolIYQ0o82PymKMgTHVM62uINYvFrvjd6O3oLfScjeBm8EXmJZG5wJAak4q3RiBEEKaoXWR/OKLL+Dv7w9LS0tYWloiICAA27Zt02VsBiHWLxblKeX4MeHBHYZ+TPgRZSllBl0gARqdSwjRnFwuR3l5OS5evIjy8nLI5XKdtu+MtOpu/eijj7BkyRIkJyfjySefBAAUFhZi/vz5+Ouvv/Dqq6/qNEh9M+GbYKRwJPZd2oeRwpEG3cWqQKNzCSGaEIlEyMnJgUQi4ZYJBAJERUXBz8+vze07K62K5Nq1a7FhwwY8//zz3LKnnnoKAwcORHp6epcrkp1RVxqdSwhpXyKRCDt37lRZLpFIsHPnTsTHxysVPk3bd2ZadbdWVFTgiSeeUFn+xBNPoKKCzkwMgWJ07qODjhR44MFd4G7wo3MJIe1LLpcjJyen2TY5OTlcV6qm7Ts7rYpkv379mvwrYseOHfD29m5zUKTtFKNzAXTa0bmEkPYnFouVukybIpFIIBaLtWrf2WnV3bps2TI888wzOHbsGHdN8vjx48jLy2uyeBL9UIzObWqeZEZUhsEPPiKEtL+amhqN2mnavrPTqkjGxcXh1KlT+Pjjj7nb0/n5+eGnn37CY489psv4SBvF+sUixieG7rhDCGmSra2tRu00bd/ZaVUkASAoKAhffvmlLmMh7cSEb4IwzzB9h0EIMUBCoRACgaDZLlSBQAChUKhV+85Oq2uS+/btw4EDB1SWHzhwAPv3729zUIQQQjoGn89HVFRUs22ioqLA5/O1at/ZaZXFokWLIJOp3qmFMYZFixa1OShCCCEdx8/PD/Hx8RAIBErLBQJBk9M5NG3fmWnV3XrlyhUMGDBAZbmvry+uXr3a6u2sWLEC2dnZ+PXXX2FpaYknnngCq1atgo+PD9fm/v37eO211/D111+jvr4ekZGRWL9+PZydnbUJnRBCSBP8/Pzg4+MDsViMmpoa2NraQigUqj0j1LR9Z6VVNnZ2dvj9999Vll+9elXlsVnNOXr0KJKSknDy5Enk5uZCKpVi3LhxuHv3Ltfm1Vdfxffff49du3bh6NGjuHHjBmJjaVQmIYToGp/Ph6enJ/z9/eHp6dliwdO0fWek1ZlkTEwMUlNTsWfPHvTt2xfAgwL52muv4amnnmr1dh6dkLplyxY4OTmhuLgYoaGhqK6uxqZNm5CVlYXRo0cDADZv3gw/Pz+cPHkSjz/+uDbhE0IIIa2iVZFcvXo1oqKi4OvrCzc3NwDAtWvXEBoaiv/85z9aB1NdXQ0AcHBwAAAUFxdDKpUiIiKCa+Pr6wuhUIiioqImi2R9fT3q6+u514oRWFKpFFKpVOvYFO9tyzY6E2PLF6CcjYGx5QtQzo8u0xSPafmcK8YYcnNzcf78eVhaWmLw4MEICdH+FmdyuRxPPfUU7ty5g8LCQgBAVlYWZs6cqVT0AGD48OEIDw/HqlWrVLaTnp6OZcuWqSzPysqClZWV1vERQgjpvOrq6pCQkIDq6mqVAUfN0ehMsqioCFVVVZg0aRJ4PB7GjRuHiooKLF26FHV1dZg8eTLWrl0Lc3NzjRNISkrCpUuXuAKprcWLFyMtLY17LZFI4O7ujnHjxmn0xTxKKpUiNzcXY8eOhampaZti7AyMLV+AcjaGnI0tX4ByVuTc0q301NGoSC5fvhxhYWGYNGkSAODixYuYO3cuZsyYAT8/P3zwwQfo1asX0tPTNQoiOTkZP/zwA44dO8Z13wKAi4sLGhoacOfOHdjb23PLb968CRcXlya3ZW5u3mSRNjU11ckBoqvtdBbGli9AORsDY8sXoJy1zV2joUjnzp3DmDFjuNdff/01hg8fjk8//RRpaWlYs2aNRvduZYwhOTkZe/bsweHDh+Hl5aW0PigoCKampsjLy+OWlZSUQCwWIzg4WJPQCSGEEI1pdCb5zz//KM1PPHr0KMaPH8+9HjZsGK5du9bq7SUlJSErKwvffvstbG1tUVlZCeDBFBNLS0vY2dlh9uzZSEtLg4ODAwQCAV5++WUEBwfTyFZCCCHtTqMzSWdnZ5SVlQEAGhoa8PPPPysVq5qaGo1OaTds2IDq6mqEhYXB1dWV+9mxYwfX5uOPP8akSZMQFxeH0NBQuLi4IDs7W5OwCSGEEK1odCY5YcIELFq0CKtWrcLevXthZWWlNKL1woUL3LzJ1mjNwFoLCwusW7cO69at0yRUQgghpM00KpLvvPMOYmNjMWrUKNjY2GDr1q0wMzPj1n/++ecYN26czoMkhBBC9EGjItmzZ08cO3YM1dXVsLGxgYmJ8jMJd+3aBRsbG50GSAghhOiLVnfcsbOza3K54k45hBBCSFfQ9e5GSwghhOgIFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKKGXovksWPHEB0djV69eoHH42Hv3r1K6xljePvtt+Hq6gpLS0tERETgypUr+gmWEEKI0dFrkbx79y4GDx6MdevWNbl+9erVWLNmDTZu3IhTp07B2toakZGRuH//fgdHSgghxBh10+eHjx8/HuPHj29yHWMMGRkZ+Pe//42YmBgAwBdffAFnZ2fs3bsX06ZN68hQCSGEGCG9FsnmlJWVobKyEhEREdwyOzs7jBgxAkVFRWqLZH19Perr67nXEokEACCVSiGVSrWOR/HetmyjMzG2fAHK2RgYW74A5fzoMk0ZbJGsrKwEADg7Oystd3Z25tY1ZcWKFVi2bJnK8oMHD8LKyqrNceXm5rZ5G52JseULUM7GwNjyBSjnuro6rbZhsEVSW4sXL0ZaWhr3WiKRwN3dHePGjYNAINB6u1KpFLm5uRg7dixMTU11EapBM7Z8AcrZGHI2tnwBylmRs6JXUVMGWyRdXFwAADdv3oSrqyu3/ObNmxgyZIja95mbm8Pc3FxluampqU4OEF1tp7MwtnwBytkYGFu+AOWsbe4GWyS9vLzg4uKCvLw8rihKJBKcOnUKL774on6DI4RoTCaT6f26mFQqRbdu3XD//n3IZDK9xtJRjClnMzMz8Pm6nbSh1yJZW1uLq1evcq/Lyspw7tw5ODg4QCgUIjU1Fe+++y68vb3h5eWFJUuWoFevXpg8ebL+giaEaIQxhsrKSty5c0ffoYAxBhcXF1y7dg08Hk/f4XQIY8qZz+fDy8tLp3nqtUieOXMG4eHh3GvFtcQZM2Zgy5YteP3113H37l3MmzcPd+7cwciRI5GTkwMLCwt9hUwI0ZCiQDo5OcHKykqvv6jlcjlqa2thY2Oj8zMOQ2UsOcvlcty4cQMVFRVKl+jaSq9FMiwsDIwxtet5PB6WL1+O5cuXd2BUhBBdkclkXIHs0aOHvsOBXC5HQ0MDLCwsunTBeJgx5ezo6IgbN27otFu5a39jhBC9UlyD1MX0K0JaYmZmBgBUJAkhnUtXvxZGDIPiOGuuh1JTVCQJIYQQNahIEkKIETp+/Dj8/f1hamra6hkD6enpzc5T10ZTT4Dq6BiaQ0WSEEKacPv2bbz44osQCoUwNzeHi4sLIiMjcfz4cX2HphNpaWkYMmQIysrKsGXLljZvLz09HTwer9mfplRUVKh90IUhMNibCRBCyMNkchkKxAWoqKmAq60rQoQhMOGbtNvnxcXFoaGhAVu3bkWfPn1w8+ZN5OXloaqqqt0+syOVlpZi/vz5cHNz08n2FixYgPnz53Ovhw0bhnnz5mHu3LlNtm9oaICZmRl3dzVDRWeShBCDly3KhmemJ8K3hiMhOwHhW8PhmemJbFF2u3zenTt3UFBQgFWrViE8PBweHh4YPnw4Fi9ejKeeegoAUF5eDh6Ph3Pnzim9j8fjIT8/n1v2yy+/YNKkSRAIBLC1tUVISAhKS0u59Z9//jkGDhwIc3NzuLq6Ijk5WWl7c+bMgaOjIwQCAUaPHo3z589z68+fP4/w8HDY2tpCIBAgKCgIZ86cAQD88ccfiI6ORo8ePdC7d2/4+/tj3759XNxVVVWYNWsWeDwetmzZgi1btsDe3l7pe9i7d2+rB13Z2NjAxcWF+zExMYGtrS33etq0aUhOTkZqaip69uyJyMhIAKrdrW+88Qb69+8PKysr9OnTB0uWLNHrnZqoSBJCDFq2KBtTd07Fn5I/lZZfl1zH1J1T26VQ2tjYwMbGBnv37lV69J6mrl+/jtDQUJibm+Pw4cMoLi7GrFmz0NjYCADYsGEDkpKSMG/ePFy8eBHfffcd+vXrx73/6aefxq1bt7B//34UFxcjMDAQY8aMwd9//w0AmD59Otzc3HD69GkUFxdj0aJF3D1Kk5KSUF9fj/z8fBw/fhwrVqyAjY0N3N3dUVFRAYFAgIyMDFRUVOCZZ55pw7fVelu3boWZmRmOHz+OjRs3NtnG1tYWW7ZsweXLl5GZmYlPP/0UH3/8cYfE1xTqbiWEGCyZXIaUnBQwqA7pZ2DggYfUnFTE+MTotOu1W7du2LJlC+bOnYuNGzciMDAQo0aNwrRp0xAQENDq7axbtw52dnb4+uuvueLVv39/bv27776L1157DSkpKdyyYcOGAQAKCwvx008/4datW9xDG/7zn/9g79692L17N+bNmwexWIyFCxfC19cXAODt7c1tRywWIy4uDv7+/pBIJAgICOBuJuDi4gIejwc7O7sO7e709vbG6tWrm23z73//m/u3p6cnFixYgK+//hqvv/56e4fXJDqTJIQYrAJxgcoZ5MMYGK5JrqFAXKDzz46Li8ONGzfw3XffISoqCvn5+QgMDNRokMu5c+cQEhLS5BMobt26hRs3bmDMmDFNvvf8+fOora1Fjx49uDNbGxsblJWVcd21aWlpmDNnDiIiIrBy5UqlbtxXXnkF7777LkJCQrBixQpcuHBBsy+gHQQFBbXYZseOHXjyySfh4uICGxsb/Pvf/4ZYLO6A6JpGRZIQYrAqaip02k5TFhYWGDt2LJYsWYITJ04gMTERS5cuBQDurOzhieuPXjuztLRUu+3m1gEPHgDh6uqKc+fOKf2UlJRg4cKFAB6MKP3ll18wceJEHD58GAMGDMCePXsAAHPmzMHvv/+O6dOn4/Llyxg+fDjWrl2r9vP4fL7KJHxdXwu0trZudn1RURGmT5+OCRMm4IcffsDZs2fx1ltvoaGhQadxaIKKJCHEYLnatu5G1a1t11YDBgzA3bt3ATy4TyjwYAqDwsODeAAgICAABQUFTRYbW1tbeHp6Ii8vr8nPCgwMRGVlJbp164Z+/fop/fTs2ZNr179/f7z66qs4ePAgYmNjsXnzZm6du7s75s+fj23btiEtLQ2ffvqp2twcHR1RU1PD5ddUPu3txIkT8PDwwFtvvYWhQ4fC29sbf/zxR4fG8CgqkoQQgxUiDIGbwA08ND3Ckgce3AXuCBGG6PRzq6qqMHr0aHz55Ze4cOECysrKsGvXLqxevRoxMTEAHpwJPv7441i5ciVEIhGOHj2qdD0NAJKTkyGRSDBt2jScOXMGV65cwbZt21BSUgLgwZnghx9+iDVr1uDKlSv4+eefubO9iIgIBAcHY/LkyTh48CDKy8tx4sQJvPXWWzhz5gzu3buH5ORk5Ofn448//sDx48dx+vRp+Pn5AQBSU1Nx4MABlJWV4fz588jPz+fWNWXEiBGwsrLCm2++idLSUmRlZelk/qQmvL29IRaL8fXXX6O0tBRr1qzhzoz1hYokIcRgmfBNkBmVCQAqhVLxOiMqQ+fzJW1sbDBixAh8/PHHCA0NxaBBg7BkyRLMnTsXn3zyCdfu888/R2NjI4KCgrjn3z6sR48eOHz4MGprazFq1CgEBQXh008/5a5RzpgxAxkZGVi/fj0GDhyISZMm4cqVKw/y4/Gwb98+hIaGYubMmejfvz+mTZuGP/74A87OzjAxMUFVVRWef/559O/fH/Hx8Rg/fjyWLVsG4MFNvpOSkjBw4EBMnToV3t7eWL9+vdqcHRwc8OWXX2Lfvn3w9/fH9u3bkZ6ertPvtSVPPfUUXn31VSQnJ2PIkCE4ceIElixZ0qExPIrHdHknWAMkkUhgZ2eH6upqCAQCrbcjlUqxb98+TJgwocmL8F2NseULUM7tkfP9+/dRVlYGLy+vNj0HNluUjZScFKVBPO4Cd2REZSDWL7bV25HL5ZBIJBAIBF3+sVEKxpSz4nhzc3PD4cOHlY5rbWsBTQEhhBi8WL9YxPjEdOgddwgBqEgSQjoJE74JwjzD9B0GMTJd+9ybEEIIaQMqkoQQQogaVCQJIYQQNahIEkIIIWpQkSSEEELUoCJJCCGEqEFFkhBCCFGDiiQhhHQRnp6eyMjI0Nn2tmzZAnt7e73GoG9UJAkhpAmJiYng8Xjg8XgwMzNDv379sHz5cjQ2Nuo7NK306dOHy6epn8TERJX3PPPMM/jtt986PlgDQnfcIYR0CnK5HGKxGDU1NbC1tYVQKGz3e5FGRUVh8+bNqK+vx759+5CUlARTU1MsXrxYpW1DQwPMzMzaNZ62OHXqFPe8yBMnTiAuLg4lJSXcfUwffb6lVCqFpaVli8+97OroTJIQYvBEIhEyMzOxdetWZGdnY+vWrcjMzIRIJGrXzzU3N4eLiws8PDzw4osvIiIiAt999x2AB2eakydPxnvvvYdevXrBx8cHwIOnd+zdu1dpO/b29txjp8rLy8Hj8ZCdnY3w8HBYWVlh8ODBKCoqUnpPYWEhQkJCYGlpCXd3d7zyyitKz3q8desWoqOjYWlpCS8vL3z11VfN5uLo6AgXFxe4uLjAwcEBAODk5AQXFxfcv38f9vb22LFjB0aNGgULCwt89dVXKt2tpaWliImJgbOzM2xsbDBs2DAcOnRIm6+206AiSQgxaCKRCDt37oREIlFaLpFIsHPnznYvlA+ztLREQ0MD9zovLw8lJSXIzc3FDz/8oNG23nrrLSxYsADnzp1D//798eyzz3JduaWlpYiKikJcXBwuXLiAHTt2oLCwEMnJydz7ExMTce3aNRw5cgS7d+/G+vXrcevWrTblt2jRIqSkpEAkEiEyMlJlfW1tLSZMmIC8vDycPXsWUVFRiI6OhlgsbtPnGjLqbiWEGCy5XI6cnJxm2+Tk5MDHx6ddu14ZY8jLy8OBAwfw8ssvc8utra3x2WefadXNumDBAkycOBEAsGzZMgwcOBBXr16Fr68vVqxYgenTpyM1NRXAg4cRr1mzBqNGjcKGDRsgFouxf/9+/PTTTxg2bBgAYNOmTc0+VLk1UlNTERur/tFjgwcPxuDBg7nX77zzDvbs2YPvvvtOqYB3JVQkCSEGSywWq5xBPkoikUAsFsPT01Pnn//DDz/AxsYGUqkUcrkcCQkJSg8i9vf31/o6ZEBAAPdvV1dXAA+6UH19fXH+/HlcuHBBqQuVMQa5XI6ysjL89ttv6NatG4KCgrj1vr6+Go9EfdTQoUObXV9bW4v09HT8+OOPqKioQGNjI+7du0dnkoQQog81NTU6baep8PBwbNiwAWZmZujVqxe6dVP+lWltba3yHh6Ph0efZS+VSlXaPfyQax6PB+DBmTPwoBi98MILeOWVV1TeJxQK223EaVP5PGzBggXIzc3Ff/7zH/Tr1w+WlpaYOnWqUhd0V0NFkhBisGxtbXXaTlPW1tbo16+fRu9xdHRERUUF9/rKlSuoq6vTaBuBgYG4fPmy2s/29fVFY2MjiouLue7WkpIS3LlzR6PP0dTx48eRmJiIKVOmAHhQzMvLy9v1M/WNBu4QQgyWUCjkpiioIxAIIBQKOyiilo0ePRqffPIJzp49izNnzmD+/PlKZ42t8cYbb+DEiRNITk7GuXPncOXKFXz77bfcdT8fHx9ERUXhhRdewKlTp1BcXIw5c+a0+3QNb29vZGdn49y5czh//jwSEhK4s9+uiookaTO5XI7y8nJcvHgR5eXlXf4/Dek4fD4fUVFRzbaJiopq9/mSmvjwww/h7u6OkJAQJCQkYMGCBbCystJoGwEBATh69Ch+++03hISE4LHHHsPbb7+NXr16cW02b96MXr16YdSoUYiNjcW8efPg5OSk63SUfPTRR+jevTueeOIJREdHIzIyEoGBge36mfrGY492nhugdevW4YMPPkBlZSUGDx6MtWvXYvjw4a16r0QigZ2dHaqrq1v8i7Q5UqkU+/btw4QJEzT+q7Azam2+IpEIOTk5SoMrBAIBoqKi2jzSrqMZ2z4G2j/n+/fvo6ysDF5eXrCwsNB6O7o6zuRyOSQSCQQCgUEV1vZkTDkrjjc3NzccPnxY6bjWthYY/DXJHTt2IC0tDRs3bsSIESOQkZGByMhIlJSUtPtfTaR5ivlrj1LMX4uPj+90hZIYJj8/P/j4+HT4HXcIMfgj7KOPPsLcuXMxc+ZMDBgwABs3boSVlRU+//xzfYdm1Fo7f426Xomu8Pl8eHp6wt/fH56enlQgSYcw6DPJhoYGFBcXK90nkc/nIyIiQuUWTgr19fWor6/nXiu6Z6RSaZPDsFtL8d62bKMzaSlfsViM2traZn9R1dbWoqyszKAGVTTH2PYx0P45S6VSbn6fIfzBpLi6pIjJGBhTznK5HIwx7s5FDx/X2h7jBn1N8saNG+jduzdOnDiB4OBgbvnrr7+Oo0eP4tSpUyrvSU9Px7Jly1SWZ2VlaXzxnBDSNt26dYOLiwvc3d0N+ubfpGtoaGjAtWvXUFlZqfK0lrq6OiQkJHS9a5KaWrx4MdLS0rjXEokE7u7uGDduXJsH7uTm5mLs2LFGMaijpXzFYjGysrJa3E5CQkKnOpM0pn0MtH/O9fX1EIvFsLa2NoinSTDGuGuaign8XZ0x5Xzv3j1YWlriiSeewLFjx5SO65bu3KSOQRfJnj17wsTEBDdv3lRafvPmTbi4uDT5HnNzc5ibm6ssNzU11ckvAV1tp7NQl6+XlxdsbGyaPfAEAgG8vLw63bUjY9vHQPvlzOfzwePxcP/+/Rbv5tIRFN2NPB6v0x2X2jKmnBsbG8Hj8biR1A8f19oe3wZdJM3MzBAUFIS8vDxMnjwZwIMdnpeX12VvpttZKOavNTW6VcHQ5q+RjmdiYgJ7e3vu6RRWVlZ6PZuRy+VoaGjA/fv3jebYNJac5XI5bt++DSsrK5iYmOhsuwZdJAEgLS0NM2bMwNChQzF8+HBkZGTg7t27mDlzpr5DM3p+fn6Ij4/vMvMkSftQ9Pq09TFOusAY47rkunrXo4Ix5czn8yEUCnWap8EXyWeeeQa3b9/G22+/jcrKSgwZMgQ5OTlwdnbWd2gENH+NtIzH48HV1RVOTk56HzkslUpx7NgxhIaGGk2XujHlbGZmBj6fr9PjzOCLJAAkJydT96oBU8xfI6Q5JiYmOu0G0zaGxsZGWFhYdPmCoWCMOesS/blPCCGEqEFFkhBCCFGDiiQhhBCiRqe4JtkWihsKaTuRVEEqlaKurg4SicQo+vWNLV+AcjaGnI0tX4ByfvRmApreZK7LF8mamhoAgLu7u54jIYQQom81NTWws7NrdXuDvnerLsjlcty4caPNt2RS3N7u2rVrbbq9XWdhbPkClLMx5Gxs+QKUsyJnxe35evXqpdEUtS5/Jsnn8+Hm5qaz7QkEAqM50ADjyxegnI2BseULUM4ANDqDVKCBO4QQQogaVCQJIYQQNahItpK5uTmWLl3a5BNGuiJjyxegnI2BseULUM5t1eUH7hBCCCHaojNJQgghRA0qkoQQQogaVCQJIYQQNahIEkIIIWpQkWyFdevWwdPTExYWFhgxYgR++uknfYfUbtLT08Hj8ZR+fH199R2WTh07dgzR0dHo1asXeDwe9u7dq7SeMYa3334brq6usLS0REREBK5cuaKfYHWgpXwTExNV9nlUVJR+gtWRFStWYNiwYbC1tYWTkxMmT56MkpISpTb3799HUlISevToARsbG8TFxeHmzZt6irhtWpNvWFiYyn6eP3++niJuuw0bNiAgIIC7YUBwcDD279/PrdfV/qUi2YIdO3YgLS0NS5cuxc8//4zBgwcjMjISt27d0ndo7WbgwIGoqKjgfgoLC/Udkk7dvXsXgwcPxrp165pcv3r1aqxZswYbN27EqVOnYG1tjcjISNy/f7+DI9WNlvIFgKioKKV9vn379g6MUPeOHj2KpKQknDx5Erm5uZBKpRg3bhzu3r3LtXn11Vfx/fffY9euXTh69Chu3LiB2NhYPUatvdbkCwBz585V2s+rV6/WU8Rt5+bmhpUrV6K4uBhnzpzB6NGjERMTg19++QWADvcvI80aPnw4S0pK4l7LZDLWq1cvtmLFCj1G1X6WLl3KBg8erO8wOgwAtmfPHu61XC5nLi4u7IMPPuCW3blzh5mbm7Pt27frIULdejRfxhibMWMGi4mJ0Us8HeXWrVsMADt69Chj7ME+NTU1Zbt27eLaiEQiBoAVFRXpK0ydeTRfxhgbNWoUS0lJ0V9QHaB79+7ss88+0+n+pTPJZjQ0NKC4uBgRERHcMj6fj4iICBQVFekxsvZ15coV9OrVC3369MH06dMhFov1HVKHKSsrQ2VlpdI+t7Ozw4gRI7r0Ps/Pz4eTkxN8fHzw4osvoqqqSt8h6VR1dTUAwMHBAQBQXFwMqVSqtJ99fX0hFAq7xH5+NF+Fr776Cj179sSgQYOwePFi1NXV6SM8nZPJZPj6669x9+5dBAcH63T/dvkbnLfFX3/9BZlMBmdnZ6Xlzs7O+PXXX/UUVfsaMWIEtmzZAh8fH1RUVGDZsmUICQnBpUuXYGtrq+/w2l1lZSUANLnPFeu6mqioKMTGxsLLywulpaV48803MX78eBQVFcHExETf4bWZXC5HamoqnnzySQwaNAjAg/1sZmYGe3t7pbZdYT83lS8AJCQkwMPDA7169cKFCxfwxhtvoKSkBNnZ2XqMtm0uXryI4OBg3L9/HzY2NtizZw8GDBiAc+fO6Wz/UpEkSsaPH8/9OyAgACNGjICHhwd27tyJ2bNn6zEy0l6mTZvG/dvf3x8BAQHo27cv8vPzMWbMGD1GphtJSUm4dOlSl7u2ro66fOfNm8f929/fH66urhgzZgxKS0vRt2/fjg5TJ3x8fHDu3DlUV1dj9+7dmDFjBo4eParTz6Du1mb07NkTJiYmKiOibt68CRcXFz1F1bHs7e3Rv39/XL16Vd+hdAjFfjXmfd6nTx/07NmzS+zz5ORk/PDDDzhy5IjSI/NcXFzQ0NCAO3fuKLXv7PtZXb5NGTFiBAB06v1sZmaGfv36ISgoCCtWrMDgwYORmZmp0/1LRbIZZmZmCAoKQl5eHrdMLpcjLy8PwcHBeoys49TW1qK0tBSurq76DqVDeHl5wcXFRWmfSyQSnDp1ymj2+Z9//omqqqpOvc8ZY0hOTsaePXtw+PBheHl5Ka0PCgqCqamp0n4uKSmBWCzulPu5pXybcu7cOQDo1Pv5UXK5HPX19brdv7odW9T1fP3118zc3Jxt2bKFXb58mc2bN4/Z29uzyspKfYfWLl577TWWn5/PysrK2PHjx1lERATr2bMnu3Xrlr5D05mamhp29uxZdvbsWQaAffTRR+zs2bPsjz/+YIwxtnLlSmZvb8++/fZbduHCBRYTE8O8vLzYvXv39By5dprLt6amhi1YsIAVFRWxsrIydujQIRYYGMi8vb3Z/fv39R261l588UVmZ2fH8vPzWUVFBfdTV1fHtZk/fz4TCoXs8OHD7MyZMyw4OJgFBwfrMWrttZTv1atX2fLly9mZM2dYWVkZ+/bbb1mfPn1YaGioniPX3qJFi9jRo0dZWVkZu3DhAlu0aBHj8Xjs4MGDjDHd7V8qkq2wdu1aJhQKmZmZGRs+fDg7efKkvkNqN8888wxzdXVlZmZmrHfv3uyZZ55hV69e1XdYOnXkyBEGQOVnxowZjLEH00CWLFnCnJ2dmbm5ORszZgwrKSnRb9Bt0Fy+dXV1bNy4cczR0ZGZmpoyDw8PNnfu3E7/R2BT+QJgmzdv5trcu3ePvfTSS6x79+7MysqKTZkyhVVUVOgv6DZoKV+xWMxCQ0OZg4MDMzc3Z/369WMLFy5k1dXV+g28DWbNmsU8PDyYmZkZc3R0ZGPGjOEKJGO627/0qCxCCCFEDbomSQghhKhBRZIQQghRg4okIYQQogYVSUIIIUQNKpKEEEKIGlQkCSGEEDWoSBJCCCFqUJEkhBBC1KAiSUgHSU9Px5AhQzR6D4/Hw969e9slHl0oLy8Hj8fj7gNKSFdDRZIQLfB4vGZ/0tPTVd6zYMECpRsu60JiYiJ4PB5WrlyptHzv3r3g8Xg6/SxCjBE9T5IQLVRUVHD/3rFjB95++22UlJRwy2xsbLh/M8Ygk8lgY2OjtFxXLCwssGrVKrzwwgvo3r27zrevDw0NDTAzM9N3GITQmSQh2nBxceF+7OzswOPxuNe//vorbG1tsX//fgQFBcHc3ByFhYUq3a2nT5/G2LFj0bNnT9jZ2WHUqFH4+eefNY4lIiICLi4uWLFihdo2TXX1ZmRkwNPTk3udmJiIyZMn4/3334ezszPs7e2xfPlyNDY2YuHChXBwcICbmxs2b96ssv1ff/0VTzzxBCwsLDBo0CCVB99eunQJ48ePh42NDZydnfHcc8/hr7/+4taHhYUhOTkZqamp6NmzJyIjIzX+HghpD1QkCWknixYtwsqVKyESiRAQEKCyvqamBjNmzEBhYSFOnjwJb29vTJgwATU1NRp9jomJCd5//32sXbsWf/75Z5tiPnz4MG7cuIFjx47ho48+wtKlSzFp0iR0794dp06dwvz58/HCCy+ofM7ChQvx2muv4ezZswgODkZ0dDSqqqoAAHfu3MHo0aPx2GOP4cyZM8jJycHNmzcRHx+vtI2tW7fCzMwMx48fx8aNG9uUByG6QkWSkHayfPlyjB07Fn379oWDg4PK+tGjR+Nf//oXfH194efnh//973+oq6tTOQtrjSlTpmDIkCFYunRpm2J2cHDAmjVr4OPjg1mzZsHHxwd1dXV488034e3tjcWLF8PMzAyFhYVK70tOTkZcXBz8/PywYcMG2NnZYdOmTQCATz75BI899hjef/99+Pr64rHHHsPnn3+OI0eO4LfffuO24e3tjdWrV8PHxwc+Pj5tyoMQXaEiSUg7GTp0aLPrb968iblz58Lb2xt2dnYQCASora2FWCzW6vNWrVqFrVu3QiQSafV+ABg4cCD4/P/7teDs7Ax/f3/utYmJCXr06IFbt24pve/hp71369YNQ4cO5eI4f/48jhw5wl2TtbGxga+vLwCgtLSUe19QUJDWcRPSXmjgDiHtxNrautn1M2bMQFVVFTIzM+Hh4QFzc3MEBwejoaFBq88LDQ1FZGQkFi9ejMTERKV1fD4fjz46ViqVqmzD1NRU6TWPx2tymVwub3VctbW1iI6OxqpVq1TWubq6cv9u6fsiRB+oSBKiJ8ePH8f69esxYcIEAMC1a9eUBrNoY+XKlRgyZIhKd6WjoyMqKyvBGOOmhuhybuPJkycRGhoKAGhsbERxcTGSk5MBAIGBgfjmm2/g6emJbt3oVw7pXKi7lRA98fb2xrZt2yASiXDq1ClMnz4dlpaWbdqmv78/pk+fjjVr1igtDwsLw+3bt7F69WqUlpZi3bp12L9/f5s+62Hr1q3Dnj178OuvvyIpKQn//PMPZs2aBQBISkrC33//jWeffRanT59GaWkpDhw4gJkzZ0Imk+ksBkLaAxVJQvRk06ZN+OeffxAYGIjnnnsOr7zyCpycnNq83eXLl6t0h/r5+WH9+vVYt24dBg8ejJ9++gkLFixo82cprFy5EitXrsTgwYNRWFiI7777Dj179gQA9OrVC8ePH4dMJsO4cePg7++P1NRU2NvbK13/JMQQ8dijFyoIIYQQAoDOJAkhhBC1qEgSQgghalCRJIQQQtSgIkkIIYSoQUWSEEIIUYOKJCGEEKIGFUlCCCFEDSqShBBCiBpUJAkhhBA1qEgSQgghalCRJIQQQtT4f/vMJmwbuC7rAAAAAElFTkSuQmCC",- "text/plain": [- "<Figure size 500x300 with 1 Axes>"- ]- },- "metadata": {},- "output_type": "display_data"- "name": "stdout",- "output_type": "stream",- "text": [- "DSPy_optimizer_cache/cache/compiler\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 5/500 [00:00<00:08, 57.26it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 6 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 0%| | 2/500 [00:00<00:09, 54.86it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 3 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 0%| | 1/500 [00:00<00:09, 51.38it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 2 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 0%| | 1/500 [00:00<00:09, 51.48it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 2 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 0%| | 2/500 [00:00<00:09, 53.30it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 3 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 4/500 [00:00<00:08, 56.01it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 5 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 4/500 [00:00<00:08, 56.18it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 5 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 6/500 [00:00<00:08, 55.90it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 7 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 3/500 [00:00<00:09, 54.17it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 4 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "[I 2024-02-26 21:34:37,557] A new study created in memory with name: no-name-61921210-557c-461c-aca6-e0a8a107330a\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Starting trial #0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.76it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:15<00:00, 6.63it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 19 / 100 (19.0): 100%|██████████| 100/100 [00:14<00:00, 6.67it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 19 / 100 (19.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:14<00:00, 6.70it/s]\n",- "[I 2024-02-26 21:35:54,035] Trial 0 finished with value: 26.4 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 1, '139679081593440_predictor_instruction': 5, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 3, '139679081593584_predictor_demos': 0}. Best is trial 0 with value: 26.4.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0%)\n",- "Starting trial #1\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:17<00:00, 5.76it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 24 / 100 (24.0): 100%|██████████| 100/100 [00:16<00:00, 5.95it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 24 / 100 (24.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:16<00:00, 5.97it/s]\n",- "[I 2024-02-26 21:37:19,423] Trial 1 finished with value: 31.4 and parameters: {'139679081593344_predictor_instruction': 9, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 4, '139679081593584_predictor_demos': 2}. Best is trial 1 with value: 31.4.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 31 / 100 (31.0%)\n",- "Starting trial #2\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:16<00:00, 6.16it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.15it/s]\n",- "[I 2024-02-26 21:38:42,088] Trial 2 finished with value: 32.6 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 0, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 5, '139679081593584_predictor_demos': 8}. Best is trial 2 with value: 32.6.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0%)\n",- "Starting trial #3\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:14<00:00, 6.92it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 31 / 100 (31.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.93it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:15<00:00, 6.60it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 28 / 100 (28.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 16 / 100 (16.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 16 / 100 (16.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:15<00:00, 6.56it/s]\n",- "[I 2024-02-26 21:39:57,811] Trial 3 finished with value: 27.6 and parameters: {'139679081593344_predictor_instruction': 2, '139679081593344_predictor_demos': 2, '139679081593440_predictor_instruction': 3, '139679081593440_predictor_demos': 9, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 0}. Best is trial 2 with value: 32.6.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 30 / 100 (30.0%)\n",- "Starting trial #4\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 26 / 100 (26.0): 100%|██████████| 100/100 [00:15<00:00, 6.43it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 26 / 100 (26.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.19it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 27 / 100 (27.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.44it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.26it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 23 / 100 (23.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 22 / 100 (22.0): 100%|██████████| 100/100 [00:16<00:00, 6.07it/s]\n",- "[I 2024-02-26 21:41:18,440] Trial 4 finished with value: 24.2 and parameters: {'139679081593344_predictor_instruction': 7, '139679081593344_predictor_demos': 6, '139679081593440_predictor_instruction': 1, '139679081593440_predictor_demos': 3, '139679081593584_predictor_instruction': 0, '139679081593584_predictor_demos': 2}. Best is trial 2 with value: 32.6.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 22 / 100 (22.0%)\n",- "Starting trial #5\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:15<00:00, 6.28it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.14it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 15 / 48 (31.2): 48%|████▊ | 48/100 [00:08<00:08, 6.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:16<00:00, 5.93it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 42 / 100 (42.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:16<00:00, 6.24it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n",- "[I 2024-02-26 21:42:41,196] Trial 5 finished with value: 36.6 and parameters: {'139679081593344_predictor_instruction': 5, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 4, '139679081593440_predictor_demos': 6, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 8}. Best is trial 5 with value: 36.6.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 30 / 100 (30.0%)\n",- "Starting trial #6\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 4 / 100 (4.0): 100%|██████████| 100/100 [00:16<00:00, 6.06it/s]\n",- "[I 2024-02-26 21:42:57,916] Trial 6 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 4 / 100 (4.0%)\n",- "Trial pruned.\n",- "Starting trial #7\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:17<00:00, 5.82it/s]\n",- "[I 2024-02-26 21:43:15,274] Trial 7 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0%)\n",- "Trial pruned.\n",- "Starting trial #8\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.37it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:15<00:00, 6.41it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.39it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n",- "[I 2024-02-26 21:44:34,016] Trial 8 finished with value: 37.6 and parameters: {'139679081593344_predictor_instruction': 8, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 8, '139679081593584_predictor_instruction': 2, '139679081593584_predictor_demos': 1}. Best is trial 8 with value: 37.6.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n",- "Starting trial #9\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:16<00:00, 5.98it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 15 / 36 (41.7): 35%|███▌ | 35/100 [00:07<00:15, 4.31it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:17<00:00, 5.71it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:17<00:00, 5.88it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.09it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 5.99it/s]\n",- "[I 2024-02-26 21:45:59,368] Trial 9 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n",- "Starting trial #10\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:07<00:00, 14.09it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:07<00:00, 13.72it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 14.37it/s]\n",- "[I 2024-02-26 21:46:36,422] Trial 10 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Starting trial #11\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.54it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:06<00:00, 16.38it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.60it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.14it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.29it/s]\n",- "[I 2024-02-26 21:47:07,967] Trial 11 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Starting trial #12\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.49it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 47 / 100 (47.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:05<00:00, 16.96it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 39 / 100 (39.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.26it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.58it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.55it/s]\n",- "[I 2024-02-26 21:47:40,229] Trial 12 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Starting trial #13\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 46 / 100 (46.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n",- "[I 2024-02-26 21:48:17,557] Trial 13 finished with value: 42.4 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 7, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 13 with value: 42.4.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 42 / 100 (42.0%)\n",- "Starting trial #14\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n",- "[I 2024-02-26 21:48:33,158] Trial 14 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 32 / 100 (32.0%)\n",- "Trial pruned.\n",- "Starting trial #15\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:16<00:00, 6.01it/s]\n",- "[I 2024-02-26 21:48:49,962] Trial 15 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Trial pruned.\n",- "Starting trial #16\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.80it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.69it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:11<00:00, 8.83it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 46 / 100 (46.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 24 / 52 (46.2): 52%|█████▏ | 52/100 [00:06<00:05, 8.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:11<00:00, 8.38it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 41 / 100 (41.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:11<00:00, 8.89it/s]\n",- "[I 2024-02-26 21:49:48,516] Trial 16 finished with value: 43.0 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 7, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 3}. Best is trial 16 with value: 43.0.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n",- "Starting trial #17\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:15<00:00, 6.51it/s]\n",- "[I 2024-02-26 21:50:04,053] Trial 17 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Trial pruned.\n",- "Starting trial #18\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",- "[I 2024-02-26 21:50:20,853] Trial 18 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n",- "Trial pruned.\n",- "Starting trial #19\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.87it/s]\n",- "[I 2024-02-26 21:50:35,594] Trial 19 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0%)\n",- "Trial pruned.\n",- "Starting trial #20\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",- "[I 2024-02-26 21:50:52,428] Trial 20 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 27 / 100 (27.0%)\n",- "Trial pruned.\n",- "Starting trial #21\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:06<00:00, 16.43it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:06<00:00, 16.21it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:06<00:00, 16.06it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:06<00:00, 15.99it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:06<00:00, 15.86it/s]\n",- "[I 2024-02-26 21:51:24,668] Trial 21 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n",- "Starting trial #22\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 49 / 100 (49.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 41 / 100 (41.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:07<00:00, 13.88it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 45 / 100 (45.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.73it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 35 / 100 (35.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:07<00:00, 13.77it/s]\n",- "[I 2024-02-26 21:52:02,055] Trial 22 finished with value: 41.6 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n",- "Starting trial #23\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 21 / 100 (21.0): 100%|██████████| 100/100 [00:11<00:00, 8.86it/s]\n",- "[I 2024-02-26 21:52:13,595] Trial 23 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 21 / 100 (21.0%)\n",- "Trial pruned.\n",- "Starting trial #24\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.77it/s]\n",- "[I 2024-02-26 21:52:28,612] Trial 24 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 29 / 100 (29.0%)\n",- "Trial pruned.\n",- "Starting trial #25\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 6 / 18 (33.3): 17%|█▋ | 17/100 [00:02<00:08, 10.07it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",- " response = send_hftgi_request_v01_wrapped(\n",- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:12<00:00, 8.11it/s]\n",- "[I 2024-02-26 21:52:41,142] Trial 25 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n",- "Trial pruned.\n",- "Starting trial #26\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:14<00:00, 7.01it/s]\n",- "[I 2024-02-26 21:52:55,623] Trial 26 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 36 / 100 (36.0%)\n",- "Trial pruned.\n",- "Starting trial #27\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n",- "[I 2024-02-26 21:53:11,086] Trial 27 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 33 / 100 (33.0%)\n",- "Trial pruned.\n",- "Starting trial #28\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.87it/s]\n",- "[I 2024-02-26 21:53:18,558] Trial 28 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n",- "Trial pruned.\n",- "Starting trial #29\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:13<00:00, 7.28it/s]\n",- "[I 2024-02-26 21:53:32,494] Trial 29 pruned. \n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 34 / 100 (34.0%)\n",- "Trial pruned.\n"- ]-- }-- ],++ "outputs": [],"source": [- "from dspy.teleprompt import BayesianSignatureOptimizer\n",+ "import matplotlib.pyplot as plt\n",+ "\n",+ "trial_logs = compiled_program.trial_logs\n",+ "\n",+ "# Extracting trial numbers, scores, and pruning status\n",+ "trial_numbers = list(trial_logs.keys())\n",+ "scores = [trial_logs[trial]['score'] for trial in trial_numbers]\n",+ "pruning_status = [trial_logs[trial]['pruned'] for trial in trial_numbers]\n",+ "\n",+ "# Plot setup\n",+ "plt.figure(figsize=(5, 3))\n","\n",- "# Define hyperparameters:\n",- "N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",- "trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",- "temperature = 1.0 # The temperature configured for generating new instructions\n",+ "# Plotting each point\n",+ "for trial_number, score, pruned in zip(trial_numbers, scores, pruning_status):\n",+ " if pruned:\n",+ " plt.scatter(trial_number, score, color='grey', label='Pruned Trial' if 'Pruned Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",+ " else:\n",+ " plt.scatter(trial_number, score, color='green', label='Successful Trial' if 'Successful Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n","\n",- "# Compile\n",- "eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",- "teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=False)\n",- "compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"+ "plt.xlabel('Trial Number')\n",+ "plt.ylabel('Score')\n",+ "plt.title('Trial Scores with Pruning Status')\n",+ "plt.grid(True)\n",+ "plt.legend()\n",+ "plt.show()"]},{@@@ -466,79 -1853,41 +352,36 @@@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": null,"metadata": {},-- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 215 / 500 (43.0): 100%|██████████| 500/500 [00:30<00:00, 16.32it/s]\n"- ]- },-- {-- "name": "stdout",-- "output_type": "stream",-- "text": [- "Basline program | Score: 0:\n",- "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",- "Prompt 2 Instruction: Given the fields `context`, `question`, produce the fields `answer`.\n",- "Prompt 3 Instruction: Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\n",- "\n",- "----------------\n",- "Best program after 0 trials | Score: 24.0:\n",- "Prompt 1 Instruction: Given the fact-based nature of the questions related to pop culture, history, and entertainment, with a focus on identifying specific works or individuals, accurately retrieve and synthesize relevant information.\n",- "Prompt 2 Instruction: Given the fields `context` containing information about a specific topic, and `question` containing a fact-based question, generate the field `answer` with the specific information that directly addresses the question.\n",- "Prompt 3 Instruction: For a given question and list of passages, rank the passages in order of helpfulness in answering the question, with the most helpful passage first and the least helpful last. Provide a comma-separated list of the passage numbers in the ranked order.\n",- "\n",- "Best program after 5 trials | Score: 31.0:\n",- "Prompt 1 Instruction: Given the fact-based nature of the questions related to pop culture, history, and entertainment, with a focus on identifying specific works or individuals, accurately retrieve and synthesize relevant information.\n",- "Prompt 2 Instruction: Given the fields `context`, `question`, produce the fields `answer`.\n",- "Prompt 3 Instruction: Given a fact-based question, identify the specific work, individual, or event being referenced, and provide a concise and accurate answer based on the information provided in the passages.\n",- "\n",- "Best program after 10 trials | Score: 41.4:\n",- "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",- "Prompt 2 Instruction: Given a fact-based question related to pop culture, history, or entertainment, along with a corresponding context, identify and provide a concise answer that directly corresponds to the specific question posed.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",- "\n",- "Best program after 15 trials | Score: 42.4:\n",- "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",- "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",- "\n",- "Best program after 20 trials | Score: 42.4:\n",- "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",- "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",- "\n",- "Best program after 25 trials | Score: 42.4:\n",- "Prompt 1 Instruction: Given the fields `context`, `question`, produce the fields `search_query`.\n",- "Prompt 2 Instruction: Given the context and question about a specific event or individual, generate a concise and precise answer that directly addresses the question.\n",- "Prompt 3 Instruction: Given a fact-based question related to pop culture, history, or entertainment and a list of relevant passages, identify and rank the passages in order of relevance to the question. Return a comma-separated list of the passage numbers, with the most relevant passage number first and the least relevant last.\n",- "\n"- "Average Metric: 215 / 500 (43.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 199 / 500 (39.8): 100%|██████████| 500/500 [01:21<00:00, 6.11it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 199 / 500 (39.8%)\n"-- ]-- }-- ],++ "outputs": [],"source": [- "bayesian_train_score = evaluate(compiled_program,devset=trainset)\n",- "bayesian_eval_score = evaluate(compiled_program, devset=devset)"+ "best_score = 0\n",+ "\n",+ "def get_signature(predictor):\n",+ " if (hasattr(predictor, 'extended_signature')):\n",+ " return predictor.extended_signature\n",+ " elif (hasattr(predictor, 'signature')):\n",+ " return predictor.signature\n",+ "\n",+ "print(f\"Basline program | Score: {best_score}:\")\n",+ "for i,predictor in enumerate(program.predictors()):\n",+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",+ "print() \n",+ "\n",+ "print(\"----------------\")\n",+ "\n",+ "for trial_num in compiled_program.trial_logs:\n",+ " program_score = compiled_program.trial_logs[trial_num][\"score\"]\n",+ " program_pruned = compiled_program.trial_logs[trial_num][\"pruned\"]\n",+ " if program_score > best_score and not program_pruned:\n",+ " best_score = program_score\n",+ " best_program_so_far = compiled_program.trial_logs[trial_num][\"program\"]\n",+ " if trial_num % 5 == 0:\n",+ " print(f\"Best program after {trial_num} trials | Score: {best_score}:\")\n",+ " for i,predictor in enumerate(best_program_so_far.predictors()):\n",+ " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",+ " print() "]}],