Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

adding in bayesian optimizer notebook, plus a few minor changes to bayesian optimizer

klopsahlong7e7ed0921589
diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.py
index 68d7aacf0..bca0d299d 100644
--- a/dspy/teleprompt/signature_opt_bayesian.py
+++ b/dspy/teleprompt/signature_opt_bayesian.py
@@ -65,12 +65,12 @@ Your task is to propose an instruction that will lead a good language model to p
proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
class BasicGenerateInstructionWithExamplesAndDataObservations(dspy.Signature):
- ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will also provide you with the current ``basic instruction`` that is being used for this task. I will also provide you with some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs.
+ ("""You are an instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Specifically, I will give you some ``observations`` I have made about the dataset and task, along with some ``examples`` of the expected inputs and outputs. I will also provide you with the current ``basic instruction`` that is being used for this task.
Your task is to propose a new improved instruction and prefix for the output field that will lead a good language model to perform the task well. Don't be afraid to be creative.""")
- basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
observations = dspy.InputField(desc="Observations about the dataset and task")
examples = dspy.InputField(format=dsp.passages2text, desc="Example(s) of the task")
+ basic_instruction = dspy.InputField(desc="The initial instructions before optimization")
proposed_instruction = dspy.OutputField(desc="The improved instructions for the language model")
proposed_prefix_for_output_field = dspy.OutputField(desc="The string at the end of the prompt, which will help the model start solving the task")
@@ -112,12 +112,8 @@ class BayesianSignatureOptimizer(Teleprompter):
def _print_full_program(self, program):
for i,predictor in enumerate(program.predictors()):
if self.verbose: print(f"Predictor {i}")
- if (hasattr(predictor, 'extended_signature')):
- if self.verbose: print(f"i: {predictor.extended_signature.instructions}")
- if self.verbose: print(f"p: {predictor.extended_signature.fields[-1].name}")
- else:
- if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")
- if self.verbose: print(f"p: {predictor.extended_signature1.fields[-1].name}")
+ if self.verbose: print(f"i: {self._get_signature(predictor).instructions}")
+ if self.verbose: print(f"p: {self._get_signature(predictor).fields[-1].name}")
if self.verbose: print("\n")
def _print_model_history(self, model, n=1):
@@ -162,6 +158,12 @@ class BayesianSignatureOptimizer(Teleprompter):
# Joining all the field strings
return '\n'.join(output)
+
+ def _get_signature(self, predictor):
+ if (hasattr(predictor, 'extended_signature')):
+ return predictor.extended_signature
+ elif (hasattr(predictor, 'signature')):
+ return predictor.signature
def _generate_first_N_candidates(self, module, N, view_data, view_examples, demo_candidates, devset):
candidates = {}
@@ -186,7 +188,7 @@ class BayesianSignatureOptimizer(Teleprompter):
if example["augmented"]:
if example_set_i not in example_set:
example_set[example_set_i] = []
- fields_to_use = predictor.signature.fields
+ fields_to_use = self._get_signature(predictor).fields
input_variable_names = [field.input_variable for field in fields_to_use]
example_with_only_signature_fields = {key: value for key, value in example.items() if key in input_variable_names}
example_string = self._create_example_string(fields_to_use, example_with_only_signature_fields)
@@ -200,12 +202,8 @@ class BayesianSignatureOptimizer(Teleprompter):
for predictor in module.predictors():
basic_instruction = None
basic_prefix = None
- if (hasattr(predictor, 'extended_signature')):
- basic_instruction = predictor.extended_signature.instructions
- basic_prefix = predictor.extended_signature.fields[-1].name
- else:
- basic_instruction = predictor.extended_signature1.instructions
- basic_prefix = predictor.extended_signature1.fields[-1].name
+ basic_instruction = self._get_signature(predictor).instructions
+ basic_prefix = self._get_signature(predictor).fields[-1].name
with dspy.settings.context(lm=self.prompt_model):
# Data & Examples
if view_data and view_examples:
@@ -251,6 +249,14 @@ class BayesianSignatureOptimizer(Teleprompter):
# Set up program and evaluation function
module = student.deepcopy()
evaluate = Evaluate(devset=devset, metric=self.metric, **eval_kwargs)
+
+ # In the case where the bootstrapped and labeled demos are set to 0, we'll stil bootstrap examples to use in our meta prompt
+ if max_bootstrapped_demos==0 and max_labeled_demos==0: #TODO: address case when max_bootstrapped alone is 0
+ max_bootstrapped_demos_for_candidate_gen = 1
+ max_labeled_demos_for_candidate_gen = 1 #TODO: this might only need to be 0
+ else:
+ max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos
+ max_labeled_demos_for_candidate_gen = max_labeled_demos
# Generate N few shot example sets
demo_candidates = {}
@@ -267,7 +273,7 @@ class BayesianSignatureOptimizer(Teleprompter):
rng = random.Random(i)
shuffled_devset = devset[:] # Create a copy of devset
rng.shuffle(shuffled_devset) # Shuffle the copy
- tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos, max_labeled_demos=max_labeled_demos, teacher_settings=self.teacher_settings)
+ tp = BootstrapFewShot(metric = self.metric, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen, teacher_settings=self.teacher_settings)
candidate_program = tp.compile(student=module.deepcopy(), trainset=shuffled_devset)
# Store the candidate demos
@@ -275,10 +281,14 @@ class BayesianSignatureOptimizer(Teleprompter):
if id(module_p) not in demo_candidates.keys():
demo_candidates[id(module_p)] = []
demo_candidates[id(module_p)].append(candidate_p.demos)
-
+
# Generate N candidate prompts
instruction_candidates, _ = self._generate_first_N_candidates(module, self.n, view_data, view_examples, demo_candidates, devset)
+ # Reset demo_candidates to None for our optimization if the user asked for no fewshot examples
+ if max_bootstrapped_demos==0 and max_labeled_demos==0:
+ demo_candidates = None
+
# Initialize variables to store the best program and its score
best_score = float('-inf')
best_program = None
@@ -293,20 +303,20 @@ class BayesianSignatureOptimizer(Teleprompter):
candidate_program = baseline_program.deepcopy()
# Suggest the instruction to use for our predictor
- if self.verbose: print(f"Starting trial num: {trial_num}")
+ print(f"Starting trial #{trial_num}")
trial_logs[trial_num] = {}
for p_old, p_new in zip(baseline_program.predictors(), candidate_program.predictors()):
# Get instruction candidates for our given predictor
p_instruction_candidates = instruction_candidates[id(p_old)]
- p_demo_candidates = demo_candidates[id(p_old)]
+ if demo_candidates: p_demo_candidates = demo_candidates[id(p_old)]
# Suggest the index of the instruction candidate to use in our trial
instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))
- demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))
+ if demo_candidates: demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))
trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idx
- trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx
+ if demo_candidates: trial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idx
# Get the selected instruction candidate
selected_candidate = p_instruction_candidates[instruction_idx]
@@ -314,17 +324,17 @@ class BayesianSignatureOptimizer(Teleprompter):
selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()
# Use this candidates in our program
- p_new.extended_signature.instructions = selected_instruction
- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=selected_prefix)
+ self._get_signature(p_new).instructions = selected_instruction
+ self._get_signature(p_new).fields[-1] = self._get_signature(p_new).fields[-1]._replace(name=selected_prefix)
# Get the selected demos
- selected_demos = p_demo_candidates[demos_idx]
+ if demo_candidates: selected_demos = p_demo_candidates[demos_idx]
# Use these demos in our program
- p_new.demos = selected_demos
+ if demo_candidates: p_new.demos = selected_demos
if self.verbose: print("Evaling the following program:")
- self._print_full_program(candidate_program)
+ if self.verbose: self._print_full_program(candidate_program)
trial_logs[trial_num]["program"] = candidate_program
# Evaluate with the new prompts
@@ -347,14 +357,14 @@ class BayesianSignatureOptimizer(Teleprompter):
# Handle pruning based on the intermediate value.
if trial.should_prune():
- if self.verbose: print(f"Optuna decided to prune!")
+ print(f"Trial pruned.")
trial_logs[trial_num]["score"] = curr_weighted_avg_score
trial_logs[trial_num]["pruned"] = True
trial_num += 1
raise optuna.TrialPruned()
if self.verbose: print(f"Fully evaled score: {curr_weighted_avg_score}")
- self._print_model_history(self.task_model, n=1)
+ if self.verbose: self._print_model_history(self.task_model, n=1)
score = curr_weighted_avg_score
trial_logs[trial_num]["score"] = curr_weighted_avg_score
diff --git a/examples/qa/hotpot/hotpotqa_optimized.ipynb b/examples/qa/hotpot/hotpotqa_optimized.ipynb
new file mode 100644
index 000000000..cf002054e
--- /dev/null
+++ b/examples/qa/hotpot/hotpotqa_optimized.ipynb
@@ -0,0 +1,1915 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# __DSPy Prompt Optimizers__: Optimizing instructions & fewshot examples for LM programs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Description of this notebook, what you'll learn, etc."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 0] Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First, we'll __load in the cached requests__ for this tasks, so that we don't actually need to call any LMs for this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "SyntaxError",
+ "evalue": "unterminated string literal (detected at line 11) (1838760429.py, line 11)",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;36m Cell \u001b[0;32mIn[1], line 11\u001b[0;36m\u001b[0m\n\u001b[0;31m os.environ[\"DSP_CACHEDIR\"] = f\"repo_clone_path/sub_dir\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unterminated string literal (detected at line 11)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!rm -rf DSPy_optimizer_cache\n",
+ "!git clone https://huggingface.co/kopsahlong/DSPy_optimizer_cache\n",
+ "%cd DSPy_optimizer_cache/\n",
+ "# !git checkout master\n",
+ "%cd ..\n",
+ "import os\n",
+ "repo_clone_path = 'DSPy_optimizer_cache/cache' #TODO: update this cache to just contain the runs we need!!\n",
+ "\n",
+ "# Set up the cache for this notebook\n",
+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path\n",
+ "os.environ[\"DSP_CACHEDIR\"] = f\"{repo_clone_path}/cachedir\"\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/lfs/0/kristaoo/home\n"
+ ]
+ }
+ ],
+ "source": [
+ "!echo $HOME"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will also specify the __prompt LM model__ (in this case GPT 3.5), the __task LM model__ (Llama 13B) and the retrieval model we'll be using for our task (a HotPotQA multihop retrieval task)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os \n",
+ "import dspy\n",
+ "import openai\n",
+ "import os\n",
+ "\n",
+ "### NOTE: if you'd like to run this code without a cache, you can remove these lines to configure your OPEN AI key ###\n",
+ "# os.environ['OPENAI_API_KEY'] = \"TODO: ADD YOUR OPEN AI KEY HERE\"\n",
+ "# openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
+ "# openai.api_base = \"https://api.openai.com/v1\"\n",
+ "\n",
+ "prompt_model_name = \"gpt-3.5-turbo-1106\"\n",
+ "task_model_name = \"meta-llama/Llama-2-13b-chat-hf\"\n",
+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",
+ "\n",
+ "ports = [7140, 7141, 7142, 7143] #TODO: REMOVE THIS\n",
+ "\n",
+ "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=150)\n",
+ "task_model = dspy.HFClientTGI(model=task_model_name, port=[7140, 7141, 7142, 7143], max_tokens=150)\n",
+ "\n",
+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",
+ "\n",
+ "dspy.settings.configure(rm=colbertv2, lm=task_model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 1] Define Task"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here, we'll define the program that we'd like to run, which is a multihop [...] (we can say that it was loosely inspired by a certain paper). We additionally load in the data, and define how we'd like to evaluate this task."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.\n",
+ " table = cls._concat_blocks(blocks, axis=0)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.evaluate import Evaluate\n",
+ "import re \n",
+ "from dspy.datasets import HotPotQA\n",
+ "\n",
+ "class ReturnRankedDocuments(dspy.Signature):\n",
+ " \"\"\"Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\"\"\"\n",
+ " question = dspy.InputField(desc=\"The question we're trying to answer.\")\n",
+ " context = dspy.InputField(desc=\"List of potentially related passages.\")\n",
+ " ranking = dspy.OutputField(desc=\"A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.\")\n",
+ "\n",
+ "class RankingMultiHop(dspy.Module):\n",
+ " def __init__(self, hops, num_passages_to_retrieve, max_passages_in_context):\n",
+ " super().__init__()\n",
+ " self.hops = hops\n",
+ " self.num_passages_to_retrieve = num_passages_to_retrieve\n",
+ " self.max_passages_in_context = max_passages_in_context\n",
+ " self.retrieve = dspy.Retrieve(k = self.num_passages_to_retrieve)\n",
+ " self.generate_query = dspy.ChainOfThought(\"context ,question->search_query\")\n",
+ " self.generate_answer = dspy.ChainOfThought(\"context ,question->answer\")\n",
+ " self.generate_ranking = dspy.ChainOfThought(ReturnRankedDocuments)\n",
+ " \n",
+ " def forward (self,question) :\n",
+ " context = []\n",
+ " full_context = []\n",
+ " top_context = []\n",
+ " max_passage_num = self.max_passages_in_context\n",
+ " for hop in range(self.hops):\n",
+ " # Get a new query\n",
+ " query = self.generate_query(context = context, question = question).search_query\n",
+ " # Get new passages\n",
+ " context = self.retrieve(query).passages\n",
+ " # Add these new passages to the previous top context \n",
+ " full_context = top_context + context\n",
+ " # Get the most important indices, ranked\n",
+ " most_important_indices = self.generate_ranking(question=question, context=full_context).ranking\n",
+ " indices = [int(num) for num in re.findall(r'\\d+', most_important_indices)]\n",
+ "\n",
+ " if len(indices) < max_passage_num:\n",
+ " indices = range(1,max_passage_num+1)\n",
+ "\n",
+ " valid_indices = [index-1 for index in indices if index-1 < len(context)]\n",
+ " top_indices = sorted(valid_indices, key=lambda x: x)[:max_passage_num+1]\n",
+ " most_important_context_list = [context[idx] for idx in top_indices]\n",
+ " # Save the top context\n",
+ " top_context = most_important_context_list\n",
+ "\n",
+ " return dspy.Prediction(context=context, answer=self.generate_answer(context = top_context , question = question).answer)\n",
+ "\n",
+ "program = RankingMultiHop(hops=4, num_passages_to_retrieve=5, max_passages_in_context=5)\n",
+ "\n",
+ "# Load and configure the datasets.\n",
+ "TRAIN_SIZE = 500\n",
+ "EVAL_SIZE = 500\n",
+ "\n",
+ "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)\n",
+ "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:TRAIN_SIZE]\n",
+ "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",
+ "\n",
+ "# Set up metrics\n",
+ "NUM_THREADS = 10\n",
+ "\n",
+ "metric = dspy.evaluate.answer_exact_match\n",
+ "\n",
+ "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=None)\n",
+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",
+ "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 2] Baseline Evaluation\n",
+ "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __16%__ on our trainset, and __21.4%__ on our devset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/500 [00:00<?, ?it/s]"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/lfs/0/kristaoo/miniconda3/envs/dspy_test/lib/python3.10/site-packages/joblib/memory.py:655: JobLibCollisionWarning: Possible name collisions between functions 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:-1) and 'colbertv2_get_request_v2_wrapped' (/lfs/0/kristaoo/dspy/dsp/modules/colbertv2.py:52)\n",
+ " return self._cached_call(args, kwargs)[0]\n",
+ "Average Metric: 80 / 500 (16.0): 100%|██████████| 500/500 [01:19<00:00, 6.26it/s]\n",
+ "/lfs/0/kristaoo/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+ " df = df.applymap(truncate_cell)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 80 / 500 (16.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 107 / 500 (21.4): 100%|██████████| 500/500 [01:20<00:00, 6.24it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 107 / 500 (21.4%)\n"
+ ]
+ }
+ ],
+ "source": [
+ "baseline_train_score = evaluate(program,devset=trainset)\n",
+ "baseline_eval_score = evaluate(program, devset=devset)\n",
+ "\n",
+ "#TODO: add in retrieval eval "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can also inspect a trace from this program to see what a call using this program looks like in use. #TODO"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 3] Bayesian Optimization"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### 3a] Inspecting Pre-Optimized Program"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First, because Bayesian Optimization can take a little while to run, let's load in a precompiled program to get a better understanding for what it looks like, and see how it performs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### 3a] Optimizing a Program from scratch"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we've seen what the Bayesian Prompt Optimizer can do, let's demonstrate how to train it. Note that because all of our calls to the LM are cached, this won't cost you anything, but it may take __~15 min__ to complete. [TODO: give a description of how the Bayesian Prompt Optimizer works]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "DSPy_optimizer_cache/cache/compiler\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 5/500 [00:00<00:08, 57.26it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 6 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 2/500 [00:00<00:09, 54.86it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 1/500 [00:00<00:09, 51.38it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 1/500 [00:00<00:09, 51.48it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 2/500 [00:00<00:09, 53.30it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 4/500 [00:00<00:08, 56.01it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 5 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 4/500 [00:00<00:08, 56.18it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 5 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 6/500 [00:00<00:08, 55.90it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 7 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 3/500 [00:00<00:09, 54.17it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 4 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[I 2024-02-26 21:34:37,557] A new study created in memory with name: no-name-61921210-557c-461c-aca6-e0a8a107330a\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Starting trial #0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.76it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:15<00:00, 6.63it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 19 / 100 (19.0): 100%|██████████| 100/100 [00:14<00:00, 6.67it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 19 / 100 (19.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:14<00:00, 6.70it/s]\n",
+ "[I 2024-02-26 21:35:54,035] Trial 0 finished with value: 26.4 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 1, '139679081593440_predictor_instruction': 5, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 3, '139679081593584_predictor_demos': 0}. Best is trial 0 with value: 26.4.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0%)\n",
+ "Starting trial #1\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:17<00:00, 5.76it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 24 / 100 (24.0): 100%|██████████| 100/100 [00:16<00:00, 5.95it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 24 / 100 (24.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:16<00:00, 5.97it/s]\n",
+ "[I 2024-02-26 21:37:19,423] Trial 1 finished with value: 31.4 and parameters: {'139679081593344_predictor_instruction': 9, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 4, '139679081593584_predictor_demos': 2}. Best is trial 1 with value: 31.4.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 31 / 100 (31.0%)\n",
+ "Starting trial #2\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:16<00:00, 5.94it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:16<00:00, 6.16it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.18it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:16<00:00, 6.15it/s]\n",
+ "[I 2024-02-26 21:38:42,088] Trial 2 finished with value: 32.6 and parameters: {'139679081593344_predictor_instruction': 1, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 0, '139679081593440_predictor_demos': 4, '139679081593584_predictor_instruction': 5, '139679081593584_predictor_demos': 8}. Best is trial 2 with value: 32.6.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0%)\n",
+ "Starting trial #3\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 31 / 100 (31.0): 100%|██████████| 100/100 [00:14<00:00, 6.92it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 31 / 100 (31.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.93it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0): 100%|██████████| 100/100 [00:15<00:00, 6.60it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 28 / 100 (28.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 100 (16.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 100 (16.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:15<00:00, 6.56it/s]\n",
+ "[I 2024-02-26 21:39:57,811] Trial 3 finished with value: 27.6 and parameters: {'139679081593344_predictor_instruction': 2, '139679081593344_predictor_demos': 2, '139679081593440_predictor_instruction': 3, '139679081593440_predictor_demos': 9, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 0}. Best is trial 2 with value: 32.6.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 30 / 100 (30.0%)\n",
+ "Starting trial #4\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 26 / 100 (26.0): 100%|██████████| 100/100 [00:15<00:00, 6.43it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 26 / 100 (26.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.19it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 27 / 100 (27.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.44it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0): 100%|██████████| 100/100 [00:15<00:00, 6.26it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 23 / 100 (23.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 22 / 100 (22.0): 100%|██████████| 100/100 [00:16<00:00, 6.07it/s]\n",
+ "[I 2024-02-26 21:41:18,440] Trial 4 finished with value: 24.2 and parameters: {'139679081593344_predictor_instruction': 7, '139679081593344_predictor_demos': 6, '139679081593440_predictor_instruction': 1, '139679081593440_predictor_demos': 3, '139679081593584_predictor_instruction': 0, '139679081593584_predictor_demos': 2}. Best is trial 2 with value: 32.6.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 22 / 100 (22.0%)\n",
+ "Starting trial #5\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:15<00:00, 6.28it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.14it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 15 / 48 (31.2): 48%|████▊ | 48/100 [00:08<00:08, 6.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:16<00:00, 5.93it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 42 / 100 (42.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:16<00:00, 6.24it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 30 / 100 (30.0): 100%|██████████| 100/100 [00:16<00:00, 5.96it/s]\n",
+ "[I 2024-02-26 21:42:41,196] Trial 5 finished with value: 36.6 and parameters: {'139679081593344_predictor_instruction': 5, '139679081593344_predictor_demos': 3, '139679081593440_predictor_instruction': 4, '139679081593440_predictor_demos': 6, '139679081593584_predictor_instruction': 6, '139679081593584_predictor_demos': 8}. Best is trial 5 with value: 36.6.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 30 / 100 (30.0%)\n",
+ "Starting trial #6\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 4 / 100 (4.0): 100%|██████████| 100/100 [00:16<00:00, 6.06it/s]\n",
+ "[I 2024-02-26 21:42:57,916] Trial 6 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 4 / 100 (4.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #7\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:17<00:00, 5.82it/s]\n",
+ "[I 2024-02-26 21:43:15,274] Trial 7 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #8\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.37it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:15<00:00, 6.41it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:15<00:00, 6.39it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:15<00:00, 6.45it/s]\n",
+ "[I 2024-02-26 21:44:34,016] Trial 8 finished with value: 37.6 and parameters: {'139679081593344_predictor_instruction': 8, '139679081593344_predictor_demos': 9, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 8, '139679081593584_predictor_instruction': 2, '139679081593584_predictor_demos': 1}. Best is trial 8 with value: 37.6.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n",
+ "Starting trial #9\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:16<00:00, 5.98it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 15 / 36 (41.7): 35%|███▌ | 35/100 [00:07<00:15, 4.31it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:17<00:00, 5.71it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:17<00:00, 5.88it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:16<00:00, 6.09it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 5.99it/s]\n",
+ "[I 2024-02-26 21:45:59,368] Trial 9 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n",
+ "Starting trial #10\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:07<00:00, 14.09it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:07<00:00, 13.72it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 14.37it/s]\n",
+ "[I 2024-02-26 21:46:36,422] Trial 10 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Starting trial #11\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.54it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:06<00:00, 16.38it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.60it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.14it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.29it/s]\n",
+ "[I 2024-02-26 21:47:07,967] Trial 11 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Starting trial #12\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0): 100%|██████████| 100/100 [00:06<00:00, 16.49it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 47 / 100 (47.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0): 100%|██████████| 100/100 [00:05<00:00, 16.96it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 39 / 100 (39.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:06<00:00, 16.26it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:06<00:00, 16.58it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:06<00:00, 16.55it/s]\n",
+ "[I 2024-02-26 21:47:40,229] Trial 12 finished with value: 40.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 2, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 9 with value: 41.2.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Starting trial #13\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:07<00:00, 13.68it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 46 / 100 (46.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.84it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:07<00:00, 13.83it/s]\n",
+ "[I 2024-02-26 21:48:17,557] Trial 13 finished with value: 42.4 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 7, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 13 with value: 42.4.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 42 / 100 (42.0%)\n",
+ "Starting trial #14\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0): 100%|██████████| 100/100 [00:15<00:00, 6.49it/s]\n",
+ "[I 2024-02-26 21:48:33,158] Trial 14 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 32 / 100 (32.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #15\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:16<00:00, 6.01it/s]\n",
+ "[I 2024-02-26 21:48:49,962] Trial 15 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #16\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.80it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:11<00:00, 8.69it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 46 / 100 (46.0): 100%|██████████| 100/100 [00:11<00:00, 8.83it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 46 / 100 (46.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 24 / 52 (46.2): 52%|█████▏ | 52/100 [00:06<00:05, 8.39it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:11<00:00, 8.38it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 41 / 100 (41.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:11<00:00, 8.89it/s]\n",
+ "[I 2024-02-26 21:49:48,516] Trial 16 finished with value: 43.0 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 7, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 3}. Best is trial 16 with value: 43.0.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n",
+ "Starting trial #17\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:15<00:00, 6.51it/s]\n",
+ "[I 2024-02-26 21:50:04,053] Trial 17 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #18\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",
+ "[I 2024-02-26 21:50:20,853] Trial 18 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #19\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:14<00:00, 6.87it/s]\n",
+ "[I 2024-02-26 21:50:35,594] Trial 19 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #20\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 27 / 100 (27.0): 100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n",
+ "[I 2024-02-26 21:50:52,428] Trial 20 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 27 / 100 (27.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #21\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:06<00:00, 16.43it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:06<00:00, 16.21it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:06<00:00, 16.06it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:06<00:00, 15.99it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:06<00:00, 15.86it/s]\n",
+ "[I 2024-02-26 21:51:24,668] Trial 21 finished with value: 41.2 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 8, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n",
+ "Starting trial #22\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 49 / 100 (49.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 41 / 100 (41.0): 100%|██████████| 100/100 [00:07<00:00, 13.61it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 41 / 100 (41.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:07<00:00, 13.88it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 45 / 100 (45.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0): 100%|██████████| 100/100 [00:07<00:00, 13.73it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 35 / 100 (35.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:07<00:00, 13.77it/s]\n",
+ "[I 2024-02-26 21:52:02,055] Trial 22 finished with value: 41.6 and parameters: {'139679081593344_predictor_instruction': 0, '139679081593344_predictor_demos': 4, '139679081593440_predictor_instruction': 9, '139679081593440_predictor_demos': 5, '139679081593584_predictor_instruction': 1, '139679081593584_predictor_demos': 4}. Best is trial 16 with value: 43.0.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n",
+ "Starting trial #23\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 21 / 100 (21.0): 100%|██████████| 100/100 [00:11<00:00, 8.86it/s]\n",
+ "[I 2024-02-26 21:52:13,595] Trial 23 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 21 / 100 (21.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #24\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0): 100%|██████████| 100/100 [00:14<00:00, 6.77it/s]\n",
+ "[I 2024-02-26 21:52:28,612] Trial 24 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29 / 100 (29.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #25\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 6 / 18 (33.3): 17%|█▋ | 17/100 [00:02<00:08, 10.07it/s]/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "/lfs/0/kristaoo/dspy/dsp/modules/hf_client.py:73: UserWarning: Persisting input arguments took 0.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
+ " response = send_hftgi_request_v01_wrapped(\n",
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:12<00:00, 8.11it/s]\n",
+ "[I 2024-02-26 21:52:41,142] Trial 25 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #26\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0): 100%|██████████| 100/100 [00:14<00:00, 7.01it/s]\n",
+ "[I 2024-02-26 21:52:55,623] Trial 26 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 36 / 100 (36.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #27\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0): 100%|██████████| 100/100 [00:15<00:00, 6.53it/s]\n",
+ "[I 2024-02-26 21:53:11,086] Trial 27 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 33 / 100 (33.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #28\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:07<00:00, 13.87it/s]\n",
+ "[I 2024-02-26 21:53:18,558] Trial 28 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n",
+ "Trial pruned.\n",
+ "Starting trial #29\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0): 100%|██████████| 100/100 [00:13<00:00, 7.28it/s]\n",
+ "[I 2024-02-26 21:53:32,494] Trial 29 pruned. \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 34 / 100 (34.0%)\n",
+ "Trial pruned.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.teleprompt import BayesianSignatureOptimizer\n",
+ "\n",
+ "# Define hyperparameters:\n",
+ "N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",
+ "trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",
+ "temperature = 1.0 # The temperature configured for generating new instructions\n",
+ "\n",
+ "# Compile\n",
+ "eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",
+ "teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=False)\n",
+ "compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 215 / 500 (43.0): 100%|██████████| 500/500 [00:30<00:00, 16.32it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 215 / 500 (43.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 199 / 500 (39.8): 100%|██████████| 500/500 [01:21<00:00, 6.11it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 199 / 500 (39.8%)\n"
+ ]
+ }
+ ],
+ "source": [
+ "bayesian_train_score = evaluate(compiled_program,devset=trainset)\n",
+ "bayesian_eval_score = evaluate(compiled_program, devset=devset)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "dspy_test",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}