DSPy: The framework for programming—not prompting—language models
Python
adding in bayesian optimizer notebook, plus a few minor changes to bayesian optimizer
klopsahlong7e7ed0921589
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.pyindex 68d7aacf0..bca0d299d 100644--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@ -65,12 +65,12 @@ Your task is to propose an instruction that will lead a good language model to pproposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs.+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")observations = dspy.InputField(desc="Observations about the dataset and task")examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")@@ -112,12 +112,8 @@ class BayesianSignatureOptimizer(Teleprompter):def _print_full_program(self, program):for i,predictor in enumerate(program.predictors()):if self.verbose: print(f"Predictor {i}")- if (hasattr(predictor, 'extended_signature')):- if self.verbose: print(f"i: {predictor.extended_signature.instructions}")- if self.verbose: print(f"p: {predictor.extended_signature.fields[-1].name}")- else:- if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")- if self.verbose: print(f"p: {predictor.extended_signature1.fields[-1].name}")+ if self.verbose: print(f"i: {self._get_signature(predictor).instructions}")+ if self.verbose: print(f"p: {self._get_signature(predictor).fields[-1].name}")if self.verbose: print("\n")def _print_model_history(self, model, n=1):@@ -162,6 +158,12 @@ class BayesianSignatureOptimizer(Teleprompter):# Joining all the field stringsreturn '\n'.join(output)++ def _get_signature(self, predictor):+ if (hasattr(predictor, 'extended_signature')):+ return predictor.extended_signature+ elif (hasattr(predictor, 'signature')):+ return predictor.signaturedef _generate_first_N_candidates(self, module, N, view_data, view_examples, demo_candidates, devset):candidates = {}@@ -186,7 +188,7 @@ class BayesianSignatureOptimizer(Teleprompter):if example["augmented"]:if example_set_i not in example_set:example_set[example_set_i] = []- fields_to_use = predictor.signature.fields+ fields_to_use = self._get_signature(predictor).fieldsinput_variable_names = [field.input_variable for field in fields_to_use]example_with_only_signature_fields = {key: value for key, value in example.items() if key in input_variable_names}example_string = self._create_example_string(fields_to_use, example_with_only_signature_fields)@@ -200,12 +202,8 @@ class BayesianSignatureOptimizer(Teleprompter):for predictor in module.predictors():basic_instruction = Nonebasic_prefix = None- if (hasattr(predictor, 'extended_signature')):- basic_instruction = predictor.extended_signature.instructions- basic_prefix = predictor.extended_signature.fields[-1].name- else:- basic_instruction = predictor.extended_signature1.instructions- basic_prefix = predictor.extended_signature1.fields[-1].name+ basic_instruction = self._get_signature(predictor).instructions+ basic_prefix = self._get_signature(predictor).fields[-1].namewith dspy.settings.context(lm=self.prompt_model):# Data & Examplesif view_data and view_examples:@@ -251,6 +249,14 @@ class BayesianSignatureOptimizer(Teleprompter):# Set up program and evaluation functionmodule = student.deepcopy()evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)++ # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt+ if max_bootstrapped_demos==0 and max_labeled_demos==0: #TODO: address case when max_bootstrapped alone is 0+ max_bootstrapped_demos_for_candidate_gen = 1+ max_labeled_demos_for_candidate_gen = 1 #TODO: this might only need to be 0+ else:+ max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos+ max_labeled_demos_for_candidate_gen = max_labeled_demos# Generate N few shot example setsdemo_candidates = {}@@ -267,7 +273,7 @@ class BayesianSignatureOptimizer(Teleprompter):rng = random.Random(i)shuffled_devset = devset[:] # Create a copy of devsetrng.shuffle(shuffled_devset) # Shuffle the copy- tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos, max_labeled_demos=max_labeled_demos, teacher_settings=self.teacher_settings)+ tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen, teacher_settings=self.teacher_settings)candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_devset)# Store the candidate demos@@ -275,10 +281,14 @@ class BayesianSignatureOptimizer(Teleprompter):if id(module_p) not in demo_candidates.keys():demo_candidates[id(module_p)] = []demo_candidates[id(module_p)].append(candidate_p.demos)-+# Generate N candidate promptsinstruction_candidates, _ = self._generate_first_N_candidates(module, self.n, view_data, view_examples, demo_candidates, devset)+ # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples+ if max_bootstrapped_demos==0 and max_labeled_demos==0:+ demo_candidates = None+# Initialize variables to store the best program and its scorebest_score = float('-inf')best_program = None@@ -293,20 +303,20 @@ class BayesianSignatureOptimizer(Teleprompter):candidate_program = baseline_program.deepcopy()# Suggest the instruction to use for our predictor- if self.verbose: print(f"Starting trial num: {trial_num}")+ print(f"Starting trial #{trial_num}")trial_logs[trial_num] = {}for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):# Get instruction candidates for our given predictorp_instruction_candidates = instruction_candidates[id(p_old)]- p_demo_candidates = demo_candidates[id(p_old)]+ if demo_candidates: p_demo_candidates = demo_candidates[id(p_old)]# Suggest the index of the instruction candidate to use in our trialinstruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))- demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))+ if demo_candidates: demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx- trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx+ if demo_candidates: trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx# Get the selected instruction candidateselected_candidate = p_instruction_candidates[instruction_idx]@@ -314,17 +324,17 @@ class BayesianSignatureOptimizer(Teleprompter):selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()# Use this candidates in our program- p_new.extended_signature.instructions = selected_instruction- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=selected_prefix)+ self._get_signature(p_new).instructions = selected_instruction+ self._get_signature(p_new).fields[-1] = self._get_signature(p_new).fields[-1]._replace(name=selected_prefix)# Get the selected demos- selected_demos = p_demo_candidates[demos_idx]+ if demo_candidates: selected_demos = p_demo_candidates[demos_idx]# Use these demos in our program- p_new.demos = selected_demos+ if demo_candidates: p_new.demos = selected_demosif self.verbose: print("Evaling the following program:")- self._print_full_program(candidate_program)+ if self.verbose: self._print_full_program(candidate_program)trial_logs[trial_num]["program"] = candidate_program# Evaluate with the new prompts@@ -347,14 +357,14 @@ class BayesianSignatureOptimizer(Teleprompter):# Handle pruning based on the intermediate value.if trial.should_prune():- if self.verbose: print(f"Optuna decided to prune!")+ print(f"Trial pruned.")trial_logs[trial_num]["score"] = curr_weighted_avg_scoretrial_logs[trial_num]["pruned"] = Truetrial_num += 1raise optuna.TrialPruned()if self.verbose: print(f"Fully evaled score: {curr_weighted_avg_score}")- self._print_model_history(self.task_model, n=1)+ if self.verbose: self._print_model_history(self.task_model, n=1)score = curr_weighted_avg_scoretrial_logs[trial_num]["score"] = curr_weighted_avg_scorediff --git a/examples/qa/hotpot/hotpotqa_optimized.ipynb b/examples/qa/hotpot/hotpotqa_optimized.ipynbnew file mode 100644index 000000000..cf002054e--- /dev/null+++ b/examples/qa/hotpot/hotpotqa_optimized.ipynb@@ -0,0 +1,1915 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# __DSPy Prompt Optimizers__: Optimizing instructions & fewshot examples for LM programs"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Description of this notebook, what you'll learn, etc."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 0] Setup"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "First, we'll __load in the cached requests__ for this tasks, so that we don't actually need to call any LMs for this notebook."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "ename": "SyntaxError",+ "evalue": "unterminated string literal (detected at line 11) (1838760429.py, line 11)",+ "output_type": "error",+ "traceback": [+ "\u001b[0;36m Cell \u001b[0;32mIn[1], line 11\u001b[0;36m\u001b[0m\n\u001b[0;31m os.environ[\"DSP_CACHEDIR\"] = f\"repo_clone_path/sub_dir\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unterminated string literal (detected at line 11)\n"+ ]+ }+ ],+ "source": [+ "!rm -rf DSPy_optimizer_cache\n",+ "!git clone https://huggingface.co/kopsahlong/DSPy_optimizer_cache\n",+ "%cd DSPy_optimizer_cache/\n",+ "# !git checkout master\n",+ "%cd ..\n",+ "import os\n",+ "repo_clone_path = 'DSPy_optimizer_cache/cache' #TODO: update this cache to just contain the runs we need!!\n",+ "\n",+ "# Set up the cache for this notebook\n",+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path\n",+ "os.environ[\"DSP_CACHEDIR\"] = f\"{repo_clone_path}/cachedir\"\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "/lfs/0/kristaoo/home\n"+ ]+ }+ ],+ "source": [+ "!echo $HOME"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We will also specify the __prompt LM model__ (in this case GPT 3.5), the __task LM model__ (Llama 13B) and the retrieval model we'll be using for our task (a HotPotQA multihop retrieval task)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ }+ ],+ "source": [+ "import os \n",+ "import dspy\n",+ "import openai\n",+ "import os\n",+ "\n",+ "### NOTE: if you'd like to run this code without a cache, you can remove these lines to configure your OPEN AI key ###\n",+ "# os.environ['OPENAI_API_KEY'] = \"TODO: ADD YOUR OPEN AI KEY HERE\"\n",+ "# openai.api_key = os.environ.get('OPENAI_API_KEY')\n",+ "# openai.api_base = \"https://api.openai.com/v1\"\n",+ "\n",+ "prompt_model_name = \"gpt-3.5-turbo-1106\"\n",+ "task_model_name = \"meta-llama/Llama-2-13b-chat-hf\"\n",+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",+ "\n",+ "ports = [7140, 7141, 7142, 7143] #TODO: REMOVE THIS\n",+ "\n",+ "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=150)\n",+ "task_model = dspy.HFClientTGI(model=task_model_name, port=[7140, 7141, 7142, 7143], max_tokens=150)\n",+ "\n",+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",+ "\n",+ "dspy.settings.configure(rm=colbertv2, lm=task_model)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 1] Define Task"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Here, we'll define the program that we'd like to run, which is a multihop [...] (we can say that it was loosely inspired by a certain paper). We additionally load in the data, and define how we'd like to evaluate this task."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.\n",+ " table = cls._concat_blocks(blocks, axis=0)\n"+ ]+ }+ ],+ "source": [+ "from dspy.evaluate import Evaluate\n",+ "import re \n",+ "from dspy.datasets import HotPotQA\n",+ "\n",+ "class ReturnRankedDocuments(dspy.Signature):\n",+ " \"\"\"Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\"\"\"\n",+ " question = dspy.InputField(desc=\"The question we're trying to answer.\")\n",+ " context = dspy.InputField(desc=\"List of potentially related passages.\")\n",+ " ranking = dspy.OutputField(desc=\"A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.\")\n",+ "\n",+ "class RankingMultiHop(dspy.Module):\n",+ " def __init__(self, hops, num_passages_to_retrieve, max_passages_in_context):\n",+ " super().__init__()\n",+ " self.hops = hops\n",+ " self.num_passages_to_retrieve = num_passages_to_retrieve\n",+ " self.max_passages_in_context = max_passages_in_context\n",+ " self.retrieve = dspy.Retrieve(k = self.num_passages_to_retrieve)\n",+ " self.generate_query = dspy.ChainOfThought(\"context ,question->search_query\")\n",+ " self.generate_answer = dspy.ChainOfThought(\"context ,question->answer\")\n",+ " self.generate_ranking = dspy.ChainOfThought(ReturnRankedDocuments)\n",+ " \n",+ " def forward (self,question) :\n",+ " context = []\n",+ " full_context = []\n",+ " top_context = []\n",+ " max_passage_num = self.max_passages_in_context\n",+ " for hop in range(self.hops):\n",+ " # Get a new query\n",+ " query = self.generate_query(context = context, question = question).search_query\n",+ " # Get new passages\n",+ " context = self.retrieve(query).passages\n",+ " # Add these new passages to the previous top context \n",+ " full_context = top_context + context\n",+ " # Get the most important indices, ranked\n",+ " most_important_indices = self.generate_ranking(question=question, context=full_context).ranking\n",+ " indices = [int(num) for num in re.findall(r'\\d+', most_important_indices)]\n",+ "\n",+ " if len(indices) < max_passage_num:\n",+ " indices = range(1,max_passage_num+1)\n",+ "\n",+ " valid_indices = [index-1 for index in indices if index-1 < len(context)]\n",+ " top_indices = sorted(valid_indices, key=lambda x: x)[:max_passage_num+1]\n",+ " most_important_context_list = [context[idx] for idx in top_indices]\n",+ " # Save the top context\n",+ " top_context = most_important_context_list\n",+ "\n",+ " return dspy.Prediction(context=context, answer=self.generate_answer(context = top_context , question = question).answer)\n",+ "\n",+ "program = RankingMultiHop(hops=4, num_passages_to_retrieve=5, max_passages_in_context=5)\n",+ "\n",+ "# Load and configure the datasets.\n",+ "TRAIN_SIZE = 500\n",+ "EVAL_SIZE = 500\n",+ "\n",+ "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)\n",+ "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:TRAIN_SIZE]\n",+ "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",+ "\n",+ "# Set up metrics\n",+ "NUM_THREADS = 10\n",+ "\n",+ "metric = dspy.evaluate.answer_exact_match\n",+ "\n",+ "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=None)\n",+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",+ "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 2] Baseline Evaluation\n",+ "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __16%__ on our trainset, and __21.4%__ on our devset."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 0/500 [00:00<?, ?it/s]"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/joblib/memory.py:655: JobLibCollisionWarning: Possible name collisions between functions 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:-1) and 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:52)\n",+ " return self._cached_call(args, kwargs)[0]\n",+ "Average Metric: 80 / 500 (16.0): 100%|██████████| 500/500 [01:19<00:00, 6.26it/s]\n",+ "/lfs/0/kristaoo/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",+ " df = df.applymap(truncate_cell)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 80 / 500 (16.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 107 / 500 (21.4): 100%|██████████| 500/500 [01:20<00:00, 6.24it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 107 / 500 (21.4%)\n"+ ]+ }+ ],+ "source": [+ "baseline_train_score = evaluate(program,devset=trainset)\n",+ "baseline_eval_score = evaluate(program, devset=devset)\n",+ "\n",+ "#TODO: add in retrieval eval "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We can also inspect a trace from this program to see what a call using this program looks like in use. #TODO"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 3] Bayesian Optimization"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "#### 3a] Inspecting Pre-Optimized Program"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "First, because Bayesian Optimization can take a little while to run, let's load in a precompiled program to get a better understanding for what it looks like, and see how it performs."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": []+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "#### 3a] Optimizing a Program from scratch"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now that we've seen what the Bayesian Prompt Optimizer can do, let's demonstrate how to train it. Note that because all of our calls to the LM are cached, this won't cost you anything, but it may take __~15 min__ to complete. [TODO: give a description of how the Bayesian Prompt Optimizer works]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "DSPy_optimizer_cache/cache/compiler\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 5/500 [00:00<00:08, 57.26it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 6 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 2/500 [00:00<00:09, 54.86it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 1/500 [00:00<00:09, 51.38it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 1/500 [00:00<00:09, 51.48it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 2/500 [00:00<00:09, 53.30it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 4/500 [00:00<00:08, 56.01it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 4/500 [00:00<00:08, 56.18it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 6/500 [00:00<00:08, 55.90it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 7 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 3/500 [00:00<00:09, 54.17it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 4 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-02-26 21:34:37,557] A new study created in memory with name: no-name-61921210-557c-461c-aca6-e0a8a107330a\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Starting trial #0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.76it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:15<00:00, 6.63it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 19 / 100 (19.0): 100%|██████████| 100/100 [00:14<00:00, 6.67it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 19 / 100 (19.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:14<00:00, 6.70it/s]\n",+ "[I 2024-02-26 21:35:54,035] Trial 0 finished with value: 26.4 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 1, '139679081593440_predictor_instruction': 5, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 3, '139679081593584_predictor_demos': 0}. Best is trial 0 with value: 26.4.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0%)\n",+ "Starting trial #1\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:17<00:00, 5.76it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 24 / 100 (24.0): 100%|██████████| 100/100 [00:16<00:00, 5.95it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 24 / 100 (24.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:16<00:00, 5.97it/s]\n",+ "[I 2024-02-26 21:37:19,423] Trial 1 finished with value: 31.4 and parameters: {'139679081593344_predictor_instruction': 9, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 4, '139679081593584_predictor_demos': 2}. Best is trial 1 with value: 31.4.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 31 / 100 (31.0%)\n",+ "Starting trial #2\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:16<00:00, 6.16it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.15it/s]\n",+ "[I 2024-02-26 21:38:42,088] Trial 2 finished with value: 32.6 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 0, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 5, '139679081593584_predictor_demos': 8}. Best is trial 2 with value: 32.6.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0%)\n",+ "Starting trial #3\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:14<00:00, 6.92it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 31 / 100 (31.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.93it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:15<00:00, 6.60it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 28 / 100 (28.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 16 / 100 (16.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 16 / 100 (16.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:15<00:00, 6.56it/s]\n",+ "[I 2024-02-26 21:39:57,811] Trial 3 finished with value: 27.6 and parameters: {'139679081593344_predictor_instruction': 2, '139679081593344_predictor_demos': 2, '139679081593440_predictor_instruction': 3, '139679081593440_predictor_demos': 9, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 0}. Best is trial 2 with value: 32.6.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 30 / 100 (30.0%)\n",+ "Starting trial #4\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 26 / 100 (26.0): 100%|██████████| 100/100 [00:15<00:00, 6.43it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 26 / 100 (26.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.19it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 27 / 100 (27.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.44it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.26it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 100 (23.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 22 / 100 (22.0): 100%|██████████| 100/100 [00:16<00:00, 6.07it/s]\n",+ "[I 2024-02-26 21:41:18,440] Trial 4 finished with value: 24.2 and parameters: {'139679081593344_predictor_instruction': 7, '139679081593344_predictor_demos': 6, '139679081593440_predictor_instruction': 1, '139679081593440_predictor_demos': 3, '139679081593584_predictor_instruction': 0, '139679081593584_predictor_demos': 2}. Best is trial 2 with value: 32.6.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 22 / 100 (22.0%)\n",+ "Starting trial #5\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:15<00:00, 6.28it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.14it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 15 / 48 (31.2): 48%|████▊ | 48/100 [00:08<00:08, 6.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:16<00:00, 5.93it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:16<00:00, 6.24it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n",+ "[I 2024-02-26 21:42:41,196] Trial 5 finished with value: 36.6 and parameters: {'139679081593344_predictor_instruction': 5, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 4, '139679081593440_predictor_demos': 6, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 8}. Best is trial 5 with value: 36.6.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 30 / 100 (30.0%)\n",+ "Starting trial #6\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 4 / 100 (4.0): 100%|██████████| 100/100 [00:16<00:00, 6.06it/s]\n",+ "[I 2024-02-26 21:42:57,916] Trial 6 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 4 / 100 (4.0%)\n",+ "Trial pruned.\n",+ "Starting trial #7\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:17<00:00, 5.82it/s]\n",+ "[I 2024-02-26 21:43:15,274] Trial 7 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0%)\n",+ "Trial pruned.\n",+ "Starting trial #8\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.37it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:15<00:00, 6.41it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.39it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n",+ "[I 2024-02-26 21:44:34,016] Trial 8 finished with value: 37.6 and parameters: {'139679081593344_predictor_instruction': 8, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 8, '139679081593584_predictor_instruction': 2, '139679081593584_predictor_demos': 1}. Best is trial 8 with value: 37.6.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n",+ "Starting trial #9\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:16<00:00, 5.98it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 15 / 36 (41.7): 35%|███▌ | 35/100 [00:07<00:15, 4.31it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:17<00:00, 5.71it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:17<00:00, 5.88it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.09it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 5.99it/s]\n",+ "[I 2024-02-26 21:45:59,368] Trial 9 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n",+ "Starting trial #10\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:07<00:00, 14.09it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:07<00:00, 13.72it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 14.37it/s]\n",+ "[I 2024-02-26 21:46:36,422] Trial 10 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Starting trial #11\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.54it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:06<00:00, 16.38it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.60it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.14it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.29it/s]\n",+ "[I 2024-02-26 21:47:07,967] Trial 11 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Starting trial #12\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.49it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 47 / 100 (47.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:05<00:00, 16.96it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 39 / 100 (39.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.26it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.58it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.55it/s]\n",+ "[I 2024-02-26 21:47:40,229] Trial 12 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Starting trial #13\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 46 / 100 (46.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n",+ "[I 2024-02-26 21:48:17,557] Trial 13 finished with value: 42.4 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 7, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 13 with value: 42.4.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0%)\n",+ "Starting trial #14\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n",+ "[I 2024-02-26 21:48:33,158] Trial 14 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 32 / 100 (32.0%)\n",+ "Trial pruned.\n",+ "Starting trial #15\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:16<00:00, 6.01it/s]\n",+ "[I 2024-02-26 21:48:49,962] Trial 15 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Trial pruned.\n",+ "Starting trial #16\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.80it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.69it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:11<00:00, 8.83it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 46 / 100 (46.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 24 / 52 (46.2): 52%|█████▏ | 52/100 [00:06<00:05, 8.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:11<00:00, 8.38it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 41 / 100 (41.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:11<00:00, 8.89it/s]\n",+ "[I 2024-02-26 21:49:48,516] Trial 16 finished with value: 43.0 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 7, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 3}. Best is trial 16 with value: 43.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n",+ "Starting trial #17\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:15<00:00, 6.51it/s]\n",+ "[I 2024-02-26 21:50:04,053] Trial 17 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Trial pruned.\n",+ "Starting trial #18\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",+ "[I 2024-02-26 21:50:20,853] Trial 18 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n",+ "Trial pruned.\n",+ "Starting trial #19\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.87it/s]\n",+ "[I 2024-02-26 21:50:35,594] Trial 19 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0%)\n",+ "Trial pruned.\n",+ "Starting trial #20\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",+ "[I 2024-02-26 21:50:52,428] Trial 20 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 27 / 100 (27.0%)\n",+ "Trial pruned.\n",+ "Starting trial #21\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:06<00:00, 16.43it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:06<00:00, 16.21it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:06<00:00, 16.06it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:06<00:00, 15.99it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:06<00:00, 15.86it/s]\n",+ "[I 2024-02-26 21:51:24,668] Trial 21 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n",+ "Starting trial #22\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 41 / 100 (41.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:07<00:00, 13.88it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 45 / 100 (45.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.73it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 35 / 100 (35.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:07<00:00, 13.77it/s]\n",+ "[I 2024-02-26 21:52:02,055] Trial 22 finished with value: 41.6 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n",+ "Starting trial #23\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 21 / 100 (21.0): 100%|██████████| 100/100 [00:11<00:00, 8.86it/s]\n",+ "[I 2024-02-26 21:52:13,595] Trial 23 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 21 / 100 (21.0%)\n",+ "Trial pruned.\n",+ "Starting trial #24\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.77it/s]\n",+ "[I 2024-02-26 21:52:28,612] Trial 24 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 29 / 100 (29.0%)\n",+ "Trial pruned.\n",+ "Starting trial #25\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 6 / 18 (33.3): 17%|█▋ | 17/100 [00:02<00:08, 10.07it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",+ " response = send_hftgi_request_v01_wrapped(\n",+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:12<00:00, 8.11it/s]\n",+ "[I 2024-02-26 21:52:41,142] Trial 25 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n",+ "Trial pruned.\n",+ "Starting trial #26\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:14<00:00, 7.01it/s]\n",+ "[I 2024-02-26 21:52:55,623] Trial 26 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 36 / 100 (36.0%)\n",+ "Trial pruned.\n",+ "Starting trial #27\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n",+ "[I 2024-02-26 21:53:11,086] Trial 27 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 33 / 100 (33.0%)\n",+ "Trial pruned.\n",+ "Starting trial #28\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.87it/s]\n",+ "[I 2024-02-26 21:53:18,558] Trial 28 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n",+ "Trial pruned.\n",+ "Starting trial #29\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:13<00:00, 7.28it/s]\n",+ "[I 2024-02-26 21:53:32,494] Trial 29 pruned. \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 34 / 100 (34.0%)\n",+ "Trial pruned.\n"+ ]+ }+ ],+ "source": [+ "from dspy.teleprompt import BayesianSignatureOptimizer\n",+ "\n",+ "# Define hyperparameters:\n",+ "N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",+ "trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",+ "temperature = 1.0 # The temperature configured for generating new instructions\n",+ "\n",+ "# Compile\n",+ "eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",+ "teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=False)\n",+ "compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## Evaluate"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 215 / 500 (43.0): 100%|██████████| 500/500 [00:30<00:00, 16.32it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 215 / 500 (43.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 199 / 500 (39.8): 100%|██████████| 500/500 [01:21<00:00, 6.11it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 199 / 500 (39.8%)\n"+ ]+ }+ ],+ "source": [+ "bayesian_train_score = evaluate(compiled_program,devset=trainset)\n",+ "bayesian_eval_score = evaluate(compiled_program, devset=devset)"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "dspy_test",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.10.10"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}