Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

additonal updates to notebook, merging in refactor changes

klopsahlongeb6cf859d27f
diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.py
index e96ba9d7e..2580e0945 100644
--- a/dspy/teleprompt/mipro_optimizer.py
+++ b/dspy/teleprompt/mipro_optimizer.py
@@ -25,7 +25,7 @@ from dspy.teleprompt import MIPROOptimizer
teleprompter = MIPROOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)
kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], trials_num=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
Note that this teleprompter takes in the following parameters:
@@ -279,7 +279,7 @@ class MIPRO(Teleprompter):
return candidates, evaluated_candidates
- def compile(self, student, *, trainset, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, view_data=True, view_examples=True, requires_permission_to_run=True, num_trials=None):
+ def compile(self, student, *, trainset, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed=42, view_data=True, view_examples=True, requires_permission_to_run=True, trials_num=None):
# Define ANSI escape codes for colors
YELLOW = '\033[93m'
BLUE = '\033[94m'
@@ -288,7 +288,7 @@ class MIPRO(Teleprompter):
random.seed(seed)
- estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P
+ estimated_task_model_calls_wo_module_calls = len(trainset) * trials_num # M * T * P
estimated_prompt_model_calls = 10 + self.n * len(student.predictors()) # num data summary calls + N * P
user_message = textwrap.dedent(f"""\
@@ -296,7 +296,7 @@ class MIPRO(Teleprompter):
Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:
- {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}
+ {YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in dev set * {BLUE}{BOLD}{trials_num}{ENDC}{YELLOW} trials * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}
{YELLOW}- Prompt Model: # data summarizer calls (max {BLUE}{BOLD}10{ENDC}{YELLOW}) + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}
{YELLOW}{BOLD}Estimated Cost Calculation:{ENDC}
@@ -307,7 +307,7 @@ class MIPRO(Teleprompter):
For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task
and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:
- {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}
+ {YELLOW}- Reducing the number of trials (`trials_num`), the size of the trainset, or the number of LM calls in your program.{ENDC}
{YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}
To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
@@ -471,7 +471,7 @@ class MIPRO(Teleprompter):
objective_function = create_objective(module, instruction_candidates, demo_candidates, evaluate, trainset)
sampler = optuna.samplers.TPESampler(seed=seed)
study = optuna.create_study(direction="maximize", sampler=sampler)
- score = study.optimize(objective_function, n_trials=num_trials)
+ score = study.optimize(objective_function, n_trials=trials_num)
if best_program is not None and self.track_stats:
best_program.trial_logs = trial_logs
diff --git a/examples/qa/hotpot/hotpotqa_optimized.ipynb b/examples/qa/hotpot/hotpotqa_optimized.ipynb
deleted file mode 100644
index c9b793a6a..000000000
--- a/examples/qa/hotpot/hotpotqa_optimized.ipynb
+++ /dev/null
@@ -1,409 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<img src=\"../../../docs/images/DSPy8.png\" alt=\"DSPy7 Image\" height=\"150\"/>\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Using __Multi-stage Instruction Proposal & Optimization (MIPRO)__ in DSPy"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### FAQ 🙋\n",
- "#### 1) How does MIPRO work?\n",
- "At a high level, the MIPRO program optimizer works by first proposing candidate fewshot example sets and instructions for each prompt in your program, and then optimizing over these fewshot example sets and instructions as hyperparameters for a specified number of trials. Each trial, the optimizer evaluates different combinations of prompts on a train set, which allows it to learn which combinations yield the best performance.\n",
- "\n",
- "#### 2) How much will MIPRO cost me to run?\n",
- "Note that __this notebook__ is free to run, because all LM calls have been cached. However, when using an optimizer on your own program, here is a breakdown of the upper bound of the number of calls to the task model and prompt model respectively:\n",
- "\n",
- "- **Task model calls**: MIPRO makes up to __O(TxPxM)__ task model calls, where T is the number of trials, P is the number of prompts in the program, and M is the size of the train set. This is because the model is evaluating the program on the train set each trial. In practice, this should be lower given that MIPRO tunes poor trials early (ie. it may stop a trial after running on the first 100 or so examples if performance is poor).\n",
- "\n",
- "- **Prompt model calls**: MIPRO makes up to N*P+10 prompt model calls, where N is the number of instruction / fewshot example set candidates to generate for each prompt, and P is the number of prompts in the program. The extra 10 calls comes from generating a summary of the data in the training set, which we use in the meta prompt to create better instructions.\n",
- "\n",
- "#### 3) How should I configure the hyperparameters?\n",
- "We have yet to run full hyperparameter sweeps with MIPRO, but based off of initial experimintation, we'd recommend the following:\n",
- "- __Trial num__: Gains can be seen after about 20-30 trials. However, 100-200 trials can help with adding on additional marginal gains.\n",
- "- __n__: This hyperparameter controls the number of candidate prompts and fewshot example sets that are generated to optimize over. With more trials and less prompts to optimize, we can set n to be higher, as we have more trials to explore different combinations of prompts. If your program has between 2-3 modules and is the `num_trials=30`, we'd recommend ~`n=10`. If n is higher (say `n=100`), then we can go higher to ~`n=15`. If you have a program with only 1 module and are keeping the program 0-shot (ie. no fewshot examples), then `num_trials` should be set to equal `n`, because each trial can explore a new instruction.\n",
- "- __Training set size__: Between 200 and 500 training examples are recommended. Increasing the training set size can help prevent overfitting, but adds to the expense to run.\n",
- "\n",
- "#### 4) What should I do if I want to reduce the cost?\n",
- "You can always update hyperparameters accordingly, such as using a smaller train set, using less trials, or using a program with less modules.\n",
- "Alternatively, one strategy would be to optimize using a cheaper task model (ie. locally hosted Llama-2), as initial experiments have shown that prompts optimized for a smaller model also transfer to working well on a larger model.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 0] Setup"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "First, we'll __load in the cached requests__ for this tasks, so that we don't actually need to call any LMs for this notebook."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from huggingface_hub import hf_hub_download\n",
- "import zipfile\n",
- "import os\n",
- "\n",
- "# Define the repository ID on Hugging Face\n",
- "repo_id = 'kopsahlong/test3'\n",
- "cache_file_path = hf_hub_download(repo_id=repo_id, filename='notebook_cache_v3.zip')\n",
- "compiled_program_file_path = hf_hub_download(repo_id=repo_id, filename='compiled_program.pickle')\n",
- "# Unzipping the file\n",
- "with zipfile.ZipFile(cache_file_path, 'r') as zip_ref:\n",
- " zip_ref.extractall(\".\")\n",
- "\n",
- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = \"notebook_cache\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# TODO: add in DSPy setup"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We will also specify the __prompt LM model__ (in this case GPT 3.5), the __task LM model__ (Llama 13B) and the retrieval model we'll be using for our task (a HotPotQA multihop retrieval task)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os \n",
- "import dspy\n",
- "import openai\n",
- "import os\n",
- "\n",
- "### NOTE: if you'd like to run this code without a cache, you can remove these lines to configure your OPEN AI key ###\n",
- "# os.environ['OPENAI_API_KEY'] = \"TODO: ADD YOUR OPEN AI KEY HERE\"\n",
- "# openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
- "# openai.api_base = \"https://api.openai.com/v1\"\n",
- "\n",
- "prompt_model_name = \"gpt-3.5-turbo-1106\"\n",
- "task_model_name = \"meta-llama/Llama-2-13b-chat-hf\"\n",
- "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",
- "\n",
- "prompt_model = dspy.OpenAI(model=prompt_model_name, max_tokens=150)\n",
- "task_model = dspy.HFClientTGI(model=task_model_name, port=[7140, 7141, 7142, 7143], max_tokens=150)\n",
- "\n",
- "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",
- "\n",
- "dspy.settings.configure(rm=colbertv2, lm=task_model)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 1] Define Task"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Here, we'll define the program that we'd like to run, which is a multihop [...] (we can say that it was loosely inspired by a certain paper). We additionally load in the data, and define how we'd like to evaluate this task."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from dspy.evaluate import Evaluate\n",
- "import re \n",
- "from dspy.datasets import HotPotQA\n",
- "\n",
- "class ReturnRankedDocuments(dspy.Signature):\n",
- " \"\"\"Given a question we are trying to answer and a list of passages, return a comma separated list of the numbers associated with each passage. These numbers should be ordered by helpfulness in answering the question, with most helpful passage number first, and the least helpful last.\"\"\"\n",
- " question = dspy.InputField(desc=\"The question we're trying to answer.\")\n",
- " context = dspy.InputField(desc=\"List of potentially related passages.\")\n",
- " ranking = dspy.OutputField(desc=\"A comma separated list of numbers corresponding to passage indices, ranked in descending order by their helpfulness in answering our question.\")\n",
- "\n",
- "class RankingMultiHop(dspy.Module):\n",
- " def __init__(self, hops, num_passages_to_retrieve, max_passages_in_context):\n",
- " super().__init__()\n",
- " self.hops = hops\n",
- " self.num_passages_to_retrieve = num_passages_to_retrieve\n",
- " self.max_passages_in_context = max_passages_in_context\n",
- " self.retrieve = dspy.Retrieve(k = self.num_passages_to_retrieve)\n",
- " self.generate_query = dspy.ChainOfThought(\"context ,question->search_query\")\n",
- " self.generate_answer = dspy.ChainOfThought(\"context ,question->answer\")\n",
- " self.generate_ranking = dspy.ChainOfThought(ReturnRankedDocuments)\n",
- " \n",
- " def forward(self,question):\n",
- " context = []\n",
- " full_context = []\n",
- " top_context = []\n",
- " max_passage_num = self.max_passages_in_context\n",
- " for hop in range(self.hops):\n",
- " # Get a new query\n",
- " query = self.generate_query(context = context, question = question).search_query\n",
- " # Get new passages\n",
- " context = self.retrieve(query).passages\n",
- " # Add these new passages to the previous top context \n",
- " full_context = top_context + context\n",
- " # Get the most important indices, ranked\n",
- " most_important_indices = self.generate_ranking(question=question, context=full_context).ranking\n",
- " indices = [int(num) for num in re.findall(r'\\d+', most_important_indices)]\n",
- "\n",
- " if len(indices) < max_passage_num:\n",
- " indices = range(1,max_passage_num+1)\n",
- "\n",
- " valid_indices = [index-1 for index in indices if index-1 < len(context)]\n",
- " top_indices = sorted(valid_indices, key=lambda x: x)[:max_passage_num+1]\n",
- " most_important_context_list = [context[idx] for idx in top_indices]\n",
- " # Save the top context\n",
- " top_context = most_important_context_list\n",
- "\n",
- " return dspy.Prediction(context=context, answer=self.generate_answer(context = top_context , question = question).answer)\n",
- "\n",
- "program = RankingMultiHop(hops=4, num_passages_to_retrieve=5, max_passages_in_context=5)\n",
- "\n",
- "# Load and configure the datasets.\n",
- "TRAIN_SIZE = 500\n",
- "EVAL_SIZE = 500\n",
- "\n",
- "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0)\n",
- "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:TRAIN_SIZE]\n",
- "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",
- "\n",
- "# Set up metrics\n",
- "NUM_THREADS = 10\n",
- "\n",
- "metric = dspy.evaluate.answer_exact_match\n",
- "\n",
- "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=None)\n",
- "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",
- "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 2] Baseline Evaluation\n",
- "Now, we'll quickly evaluate our baseline program so that we can see how the performance using the Prompt Optimizer compares. We should see performance of about __16%__ on our trainset, and __21.4%__ on our devset."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "baseline_train_score = evaluate(program,devset=trainset)\n",
- "baseline_eval_score = evaluate(program, devset=devset)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 3] Optimizing with MIPRO"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### 3a] Compile Program"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import cloudpickle as pickle\n",
- "from dspy.teleprompt import BayesianSignatureOptimizer\n",
- "\n",
- "LOAD_PRECOMPILED_PROGRAM = True\n",
- "\n",
- "# By default, we will load the precompiled program\n",
- "if LOAD_PRECOMPILED_PROGRAM:\n",
- " # Load a our precompiled program\n",
- " with open(compiled_program_file_path, 'rb') as file:\n",
- " # Load the data from the file\n",
- " compiled_program = pickle.load(file)\n",
- "# Otherwise, if desired, the program can be compiled from scratch \n",
- "else:\n",
- " # Define hyperparameters:\n",
- " N = 10 # The number of instructions and fewshot examples that we will generate and optimize over\n",
- " trials = 30 # The number of optimization trials to be run (we will test out a new combination of instructions and fewshot examples in each trial) \n",
- " temperature = 1.0 # The temperature configured for generating new instructions\n",
- "\n",
- " # Compile\n",
- " eval_kwargs = dict(num_threads=16, display_progress=True, display_table=0)\n",
- " teleprompter = BayesianSignatureOptimizer(prompt_model=prompt_model, task_model=task_model, metric=metric, n=N, init_temperature=temperature, verbose=True)\n",
- " compiled_program = teleprompter.compile(program.deepcopy(), devset=trainset, optuna_trials_num=trials, max_bootstrapped_demos=1,max_labeled_demos=2, eval_kwargs=eval_kwargs)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "compiled_program"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### 3b] Evaluate optimized program"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "bayesian_train_score = evaluate(compiled_program, devset=trainset)\n",
- "bayesian_eval_score = evaluate(compiled_program, devset=devset)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### 3c] Visualizing scores & prompts over trials"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, let's take a look at how this optimization looked over the course of each trial. We see that, in general, performance increases as trials go on, until it saturates after ~trial 13. Note that some of the 'pruned' trials have high scores, but were pruned early because they had comparitively lower scores on the easier slices of the data."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import matplotlib.pyplot as plt\n",
- "\n",
- "trial_logs = compiled_program.trial_logs\n",
- "\n",
- "# Extracting trial numbers, scores, and pruning status\n",
- "trial_numbers = list(trial_logs.keys())\n",
- "scores = [trial_logs[trial]['score'] for trial in trial_numbers]\n",
- "pruning_status = [trial_logs[trial]['pruned'] for trial in trial_numbers]\n",
- "\n",
- "# Plot setup\n",
- "plt.figure(figsize=(5, 3))\n",
- "\n",
- "# Plotting each point\n",
- "for trial_number, score, pruned in zip(trial_numbers, scores, pruning_status):\n",
- " if pruned:\n",
- " plt.scatter(trial_number, score, color='grey', label='Pruned Trial' if 'Pruned Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",
- " else:\n",
- " plt.scatter(trial_number, score, color='green', label='Successful Trial' if 'Successful Trial' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n",
- "\n",
- "plt.xlabel('Trial Number')\n",
- "plt.ylabel('Score')\n",
- "plt.title('Trial Scores with Pruning Status')\n",
- "plt.grid(True)\n",
- "plt.legend()\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We can also visualize the best prompts discovered by MIPRO as our trials progress... "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "best_score = 0\n",
- "\n",
- "def get_signature(predictor):\n",
- " if (hasattr(predictor, 'extended_signature')):\n",
- " return predictor.extended_signature\n",
- " elif (hasattr(predictor, 'signature')):\n",
- " return predictor.signature\n",
- "\n",
- "print(f\"Basline program | Score: {best_score}:\")\n",
- "for i,predictor in enumerate(program.predictors()):\n",
- " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",
- "print() \n",
- "\n",
- "print(\"----------------\")\n",
- "\n",
- "for trial_num in compiled_program.trial_logs:\n",
- " program_score = compiled_program.trial_logs[trial_num][\"score\"]\n",
- " program_pruned = compiled_program.trial_logs[trial_num][\"pruned\"]\n",
- " if program_score > best_score and not program_pruned:\n",
- " best_score = program_score\n",
- " best_program_so_far = compiled_program.trial_logs[trial_num][\"program\"]\n",
- " if trial_num % 5 == 0:\n",
- " print(f\"Best program after {trial_num} trials | Score: {best_score}:\")\n",
- " for i,predictor in enumerate(best_program_so_far.predictors()):\n",
- " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n",
- " print() "
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "dspy_test",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}