DSPy: The framework for programming—not prompting—language models
Python
Add ipynb with KNNFewShot example
Daniel Machlab07eb3eb12641
diff --git a/examples/knn.ipynb b/examples/knn.ipynbnew file mode 100644index 000000000..e54814232--- /dev/null+++ b/examples/knn.ipynb@@ -0,0 +1,1165 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# DSPy KNN few-shot example \n",+ "This noteboook shows how KNN few-shot can be implemented with DSPy using the **KNNFewShot** teleprompter. To illustrate, we use the HotPotQA dataset. Please see [intro.ipynb](../intro.ipynb) for other example use cases of DSPy.\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "import openai\n",+ "import dspy\n",+ "import json"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "with open(\"creds.json\", \"r\") as creds:\n",+ " api_key = json.loads(creds.read())[\"openai_key\"]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "lm = dspy.OpenAI(model='gpt-4', api_key=api_key, model_type='chat', max_tokens = 500)\n",+ "dspy.settings.configure(lm=lm)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.datasets import HotPotQA\n",+ "\n",+ "# Load the dataset.\n",+ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n",+ "\n",+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",+ "devset = [x.with_inputs('question') for x in dataset.dev]\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Example({'question': 'At My Window was released by which American singer-songwriter?', 'answer': 'John Townes Van Zandt'}) (input_keys={'question'})\n",+ "Question: At My Window was released by which American singer-songwriter?\n",+ "Answer: John Townes Van Zandt\n"+ ]+ }+ ],+ "source": [+ "train_example = trainset[0]\n",+ "print(train_example)\n",+ "print(f\"Question: {train_example.question}\")\n",+ "print(f\"Answer: {train_example.answer}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [],+ "source": [+ "class BasicQA(dspy.Signature):\n",+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",+ "\n",+ " question = dspy.InputField()\n",+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "class BasicQABot(dspy.Module):\n",+ " def __init__(self):\n",+ " super().__init__()\n",+ "\n",+ " self.generate = dspy.Predict(BasicQA)\n",+ "\n",+ " def forward(self,question):\n",+ " prediction = self.generate(question = question)\n",+ " return dspy.Prediction(answer = prediction.answer)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "'Alfred the Great'"+ ]+ },+ "execution_count": 8,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "qa_bot = BasicQABot()\n",+ "pred = qa_bot.forward(\"In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?\")\n",+ "pred.answer"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.teleprompt import KNNFewShot\n",+ "from dspy.predict.knn import KNN\n",+ "\n",+ "knn_teleprompter = KNNFewShot(KNN, 7, trainset)\n",+ "compiled_knn = knn_teleprompter.compile(BasicQABot(), trainset=trainset)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:02, 1.44it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n",+ "Question: Are both Cangzhou and Qionghai in the Hebei province of China?\n",+ "Expected answer: no\n",+ "Prediction: No\n"+ ]+ }+ ],+ "source": [+ "example = devset[0]\n",+ "pred = compiled_knn(question = example.question)\n",+ "print(\"Question: \", example.question)\n",+ "print(\"Expected answer: \", example.answer)\n",+ "print(\"Prediction: \", pred.answer)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Question: ${question}\n",+ "Answer: often between 1 and 5 words\n",+ "\n",+ "---\n",+ "\n",+ "Question: On the coast of what ocean is the birthplace of Diogal Sakho?\n",+ "Answer: Atlantic Ocean\n",+ "\n",+ "Question: Which is taller, the Empire State Building or the Bank of America Tower?\n",+ "Answer: Empire State Building\n",+ "\n",+ "Question: Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where?\n",+ "Answer: Space\n",+ "\n",+ "Question: Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?\n",+ "Answer: Aleem Dar\n",+ "\n",+ "Question: What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?\n",+ "Answer: Operation Citadel\n",+ "\n",+ "Question: Which of these publications was most recently published, Who Put the Bomp or Self?\n",+ "Answer: Self\n",+ "\n",+ "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",+ "Answer: Tae Kwon Do Times\n",+ "\n",+ "---\n",+ "\n",+ "Question: Are both Cangzhou and Qionghai in the Hebei province of China?\n",+ "Answer:\u001b[32m No\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ }+ ],+ "source": [+ "lm.inspect_history(1)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.28it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.06it/s] | 1/50 [00:03<03:01, 3.70s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.57it/s] | 2/50 [00:08<03:14, 4.05s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.16it/s] | 3/50 [00:11<02:58, 3.79s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.30it/s] | 4/50 [00:15<02:56, 3.84s/it] \n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.52it/s] | 5/50 [00:19<02:49, 3.78s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.10it/s] | 6/50 [00:22<02:43, 3.72s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.51it/s] | 7/50 [00:26<02:46, 3.88s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.02s/it] | 8/50 [00:30<02:34, 3.68s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.18it/s] | 9/50 [00:35<02:54, 4.26s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.31it/s] | 10/50 [00:39<02:46, 4.15s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.26it/s] | 11/50 [00:43<02:33, 3.94s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.16it/s] | 12/50 [00:47<02:35, 4.09s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.32it/s] | 13/50 [00:51<02:29, 4.04s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.62it/s] | 14/50 [00:55<02:23, 3.97s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:02, 1.37it/s] | 15/50 [00:58<02:11, 3.75s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.28it/s] | 16/50 [01:02<02:05, 3.70s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:02, 1.39it/s] | 17/50 [01:05<02:04, 3.78s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:05<00:04, 1.36s/it] | 18/50 [01:09<01:58, 3.69s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.52it/s] | 19/50 [01:15<02:16, 4.39s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.21it/s] | 20/50 [01:19<02:03, 4.13s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.27it/s] | 21/50 [01:22<01:57, 4.06s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.10it/s] | 22/50 [01:26<01:52, 4.01s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.25it/s] | 23/50 [01:31<01:50, 4.08s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.04s/it] | 24/50 [01:34<01:44, 4.04s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.08it/s] | 25/50 [01:39<01:47, 4.32s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.14it/s] | 26/50 [01:44<01:44, 4.36s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.22s/it] | 27/50 [01:49<01:43, 4.50s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.00s/it] | 28/50 [01:54<01:45, 4.77s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.21it/s] | 29/50 [01:59<01:40, 4.78s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.67it/s] | 30/50 [02:03<01:29, 4.45s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.54it/s] | 31/50 [02:06<01:19, 4.17s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.75it/s] | 32/50 [02:09<01:09, 3.84s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.53it/s] | 33/50 [02:12<01:01, 3.65s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.51it/s] | 34/50 [02:16<00:55, 3.50s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.06it/s] | 35/50 [02:20<00:54, 3.65s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.15s/it]▏ | 36/50 [02:25<00:57, 4.09s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:01, 1.57it/s]▍ | 37/50 [02:30<00:57, 4.41s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.08s/it]▌ | 38/50 [02:33<00:47, 4.00s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:02, 1.35it/s]▊ | 39/50 [02:38<00:47, 4.33s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.13it/s]█ | 40/50 [02:42<00:41, 4.20s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.30it/s]█▏ | 41/50 [02:46<00:38, 4.29s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.04s/it]█▍ | 42/50 [02:50<00:32, 4.07s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.27it/s]█▌ | 43/50 [02:56<00:31, 4.53s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.33it/s]█▊ | 44/50 [02:59<00:25, 4.28s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.04s/it]██ | 45/50 [03:03<00:20, 4.20s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.02s/it]██▏| 46/50 [03:09<00:18, 4.61s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:02<00:02, 1.35it/s]██▍| 47/50 [03:14<00:14, 4.74s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:03<00:02, 1.17it/s]██▌| 48/50 [03:17<00:08, 4.38s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 57%|█████▋ | 4/7 [00:04<00:03, 1.02s/it]██▊| 49/50 [03:21<00:04, 4.28s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 4 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 50 (46.0): 100%|██████████| 50/50 [03:26<00:00, 4.13s/it]\n",+ "/home/jovyan/scdc/project-vaqa-autosuggest/Query-Generation-exploratory/dspy/dspy/evaluate/evaluate.py:126: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",+ " df = df.applymap(truncate_cell)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 23 / 50 (46.0%)\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_aaaf4 th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_aaaf4 td {\n",+ " text-align: left;\n",+ "}\n",+ "#T_aaaf4_row0_col0, #T_aaaf4_row0_col1, #T_aaaf4_row0_col2, #T_aaaf4_row0_col3, #T_aaaf4_row0_col4, #T_aaaf4_row1_col0, #T_aaaf4_row1_col1, #T_aaaf4_row1_col2, #T_aaaf4_row1_col3, #T_aaaf4_row1_col4, #T_aaaf4_row2_col0, #T_aaaf4_row2_col1, #T_aaaf4_row2_col2, #T_aaaf4_row2_col3, #T_aaaf4_row2_col4, #T_aaaf4_row3_col0, #T_aaaf4_row3_col1, #T_aaaf4_row3_col2, #T_aaaf4_row3_col3, #T_aaaf4_row3_col4, #T_aaaf4_row4_col0, #T_aaaf4_row4_col1, #T_aaaf4_row4_col2, #T_aaaf4_row4_col3, #T_aaaf4_row4_col4 {\n",+ " text-align: left;\n",+ " white-space: pre-wrap;\n",+ " word-wrap: break-word;\n",+ " max-width: 400px;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_aaaf4\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th id=\"T_aaaf4_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",+ " <th id=\"T_aaaf4_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",+ " <th id=\"T_aaaf4_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",+ " <th id=\"T_aaaf4_level0_col3\" class=\"col_heading level0 col3\" >pred_answer</th>\n",+ " <th id=\"T_aaaf4_level0_col4\" class=\"col_heading level0 col4\" >answer_exact_match</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_aaaf4_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_aaaf4_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",+ " <td id=\"T_aaaf4_row0_col1\" class=\"data row0 col1\" >no</td>\n",+ " <td id=\"T_aaaf4_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",+ " <td id=\"T_aaaf4_row0_col3\" class=\"data row0 col3\" >No</td>\n",+ " <td id=\"T_aaaf4_row0_col4\" class=\"data row0 col4\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_aaaf4_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_aaaf4_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",+ " <td id=\"T_aaaf4_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",+ " <td id=\"T_aaaf4_row1_col2\" class=\"data row1 col2\" >{'2017–18 Pittsburgh Penguins season', '2017 NHL Expansion Draft'}</td>\n",+ " <td id=\"T_aaaf4_row1_col3\" class=\"data row1 col3\" >National Hockey League</td>\n",+ " <td id=\"T_aaaf4_row1_col4\" class=\"data row1 col4\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_aaaf4_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_aaaf4_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",+ " <td id=\"T_aaaf4_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",+ " <td id=\"T_aaaf4_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",+ " <td id=\"T_aaaf4_row2_col3\" class=\"data row2 col3\" >Steve Yzerman</td>\n",+ " <td id=\"T_aaaf4_row2_col4\" class=\"data row2 col4\" >✔️ [True]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_aaaf4_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_aaaf4_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",+ " <td id=\"T_aaaf4_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",+ " <td id=\"T_aaaf4_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",+ " <td id=\"T_aaaf4_row3_col3\" class=\"data row3 col3\" >Tyne River</td>\n",+ " <td id=\"T_aaaf4_row3_col4\" class=\"data row3 col4\" >❌ [False]</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_aaaf4_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_aaaf4_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",+ " <td id=\"T_aaaf4_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",+ " <td id=\"T_aaaf4_row4_col2\" class=\"data row4 col2\" >{'Æthelweard (son of Alfred)', 'Ealhswith'}</td>\n",+ " <td id=\"T_aaaf4_row4_col3\" class=\"data row4 col3\" >Alfred the Great</td>\n",+ " <td id=\"T_aaaf4_row4_col4\" class=\"data row4 col4\" >❌ [False]</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f2d430888b0>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/html": [+ "\n",+ " <div style='\n",+ " text-align: center; \n",+ " font-size: 16px; \n",+ " font-weight: bold; \n",+ " color: #555; \n",+ " margin: 10px 0;'>\n",+ " ... 45 more rows not displayed ...\n",+ " </div>\n",+ " "+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "text/plain": [+ "(46.0,\n",+ " question example_answer gold_titles pred_answer answer_exact_match\n",+ " 0 Are both Ca... no {Cangzhou, ... No True \n",+ " 1 Who conduct... National Ho... {2017–18 Pi... National Ho... True \n",+ " 2 The Wings e... Steve Yzerman {2006–07 De... Steve Yzerman True \n",+ " 3 What river ... the River Tyne {Crichton C... Tyne River False \n",+ " 4 In the 10th... King Alfred... {Æthelweard... Alfred the ... False \n",+ " 5 The Newark ... Port Author... {Newark Air... Port Author... True \n",+ " 6 Where did a... Bundesliga {Claudio Pi... Peru False \n",+ " 7 Are both Ch... no {Chico Muni... No True \n",+ " 8 In which Ma... Waldo Count... {Stockton S... Waldo County False \n",+ " 9 Which 90s r... The Afghan ... {Gene (band... The Afghan ... True \n",+ " 10 What year d... 79 AD {Curse of t... 79 AD True \n",+ " 11 Is the 72nd... the oldest {First Unit... Oldest True \n",+ " 12 Was Stanisl... not {Stanisław ... Yes False \n",+ " 13 Which film ... Del Lord {Wang Xiaos... Wang Xiaoshuai False \n",+ " 14 Lord North ... Jonathan Wi... {Jonathan A... Jonathan Ai... False \n",+ " 15 What is the... Marche {Marche, Po... Marche True \n",+ " 16 William Hug... 7,402 at th... {Kosciusko,... Unknown False \n",+ " 17 What do stu... design thei... {Gallatin S... Study Art False \n",+ " 18 What is the... English {Restaurant... British False \n",+ " 19 What Americ... Robert F. Chew {Robert F. ... Wood Harris False \n",+ " 20 What city i... Manchester {Toby Sawye... London False \n",+ " 21 Who was bor... Deepa Mehta {Tony Kaye ... Tony Kaye False \n",+ " 22 What is the... the good ma... {Boise Town... The Bon Marché False \n",+ " 23 Who did Liz... Christine C... {Lizzette R... Christine C... True \n",+ " 24 What was th... William Str... {P. T. Barn... Zerah Colburn False \n",+ " 25 Which battl... Battle of t... {Meuse-Argo... Battle of t... True \n",+ " 26 What cricke... Ian Botham {Ian Botham... Terry Alderman False \n",+ " 27 What is the... defensive a... {1982 NC St... Deceased False \n",+ " 28 Which Scott... Ewan McGregor {Come What ... Ewan McGregor True \n",+ " 29 Where have ... space {Frank De W... Space True \n",+ " 30 The origina... Maria Yermo... {Wild Honey... Maria Yermo... True \n",+ " 31 Are Roswell... no {Pago Pago ... No True \n",+ " 32 Untold: The... the voice o... {Marv Alber... Gus Johnson False \n",+ " 33 Are Walt Di... yes {Sacro GRA,... No False \n",+ " 34 What is the... Hamas {Status of ... Hamas True \n",+ " 35 What album ... 1989 {Wildest Dr... 1989 True \n",+ " 36 Which is co... Apera {Apera, Gun... Apera True \n",+ " 37 Do The Drum... no {Pussy Galo... No True \n",+ " 38 What is the... Exon {Banded Bro... UoE False \n",+ " 39 Are both Be... yes {Len Wisema... Yes True \n",+ " 40 Steven Cuit... Bill Melendez {Steven C. ... Bill Melendez True \n",+ " 41 Shark Creek... Clarence River {Clarence R... Clarence River True \n",+ " 42 Who was the... Pixar {Finding Do... Lindsey Col... False \n",+ " 43 Who purchas... Renault {Benetton F... Jean Todt False \n",+ " 44 Fredrick La... Cadwalader ... {Frederick ... Cadwalader ... True \n",+ " 45 Gordon Warn... \"Forza Ital... {Franco Zef... Democratic ... False \n",+ " 46 André Zucca... the Wehrmacht {André Zucc... The Propaga... False \n",+ " 47 Both Bill P... cricketer {Bill Ponsf... Cricket False \n",+ " 48 Suzana S. ... Danny Wallace {Yes Man (f... James Ellroy False \n",+ " 49 In what cit... Portland {Election L... New York False )"+ ]+ },+ "execution_count": 12,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "from dspy.evaluate.evaluate import Evaluate\n",+ "\n",+ "# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.\n",+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)\n",+ "\n",+ "# Evaluate the `compiled_knn` program with the `answer_exact_match` metric.\n",+ "metric = dspy.evaluate.answer_exact_match\n",+ "\n",+ " \n",+ "evaluate_on_hotpotqa(compiled_knn, metric)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": []+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "Python 3.9.18 64-bit ('pipenv')",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.9.18"+ },+ "orig_nbformat": 4,+ "vscode": {+ "interpreter": {+ "hash": "1d3dd57b136d599382d2667f19e9aed8a929301e79008dd54c75c965d79714ea"+ }+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}