DSPy: The framework for programming—not prompting—language models
Python
Revert back changes to intrinsic metrics
Shangyintdad446f8f264
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynbindex 845b4e240..350647a51 100644--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb@@ -1,655 +1,346 @@{- "cells": [- {- "cell_type": "code",- "execution_count": 2,- "metadata": {},- "outputs": [],- "source": [- "%load_ext autoreload\n",- "%autoreload 2\n",- "\n",- "import dspy\n",- "from dsp.utils import deduplicate\n",- "from dspy.datasets import HotPotQA\n",- "from dspy.predict.retry import Retry\n",- "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",- "from dspy.evaluate.evaluate import Evaluate\n",- "\n",- "from dspy.primitives.assertions import assert_transform_module, backtrack_handler"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "metadata": {},- "outputs": [],- "source": [- "import os\n",- "import openai\n",- "openai.api_key = os.getenv('OPENAI_API_KEY')"- ]- },- {- "cell_type": "code",- "execution_count": 4,- "metadata": {},- "outputs": [],- "source": [- "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",- "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",- "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",- "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"- ]- },- {- "cell_type": "code",- "execution_count": 5,- "metadata": {},- "outputs": [],- "source": [- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",- "trainset = [x.with_inputs('question') for x in dataset.train]\n",- "devset = [x.with_inputs('question') for x in dataset.dev]"- ]- },- {- "cell_type": "code",- "execution_count": 6,- "metadata": {},- "outputs": [],- "source": [- "# Suggestion helper functions and Teleprompter metric\n",- "\n",- "def validate_query_distinction_local(previous_queries, query):\n",- " \"\"\"check if query is distinct from previous queries\"\"\"\n",- " if previous_queries == []:\n",- " return True\n",- " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",- " return False\n",- " return True\n",- "\n",- "\n",- "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",- " if not dspy.evaluate.answer_exact_match(example, pred):\n",- " return False\n",- "\n",- " if not dspy.evaluate.answer_passage_match(example, pred):\n",- " return False\n",- "\n",- " return True\n"- ]- },- {- "cell_type": "code",- "execution_count": 7,- "metadata": {},- "outputs": [],- "source": [- "# Extrinsic metrics\n",- "\n",- "def gold_passages_retrieved(example, pred, trace=None):\n",- " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",- " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",- "\n",- " return gold_titles.issubset(found_titles)"- ]- },- {- "cell_type": "code",- "execution_count": 8,- "metadata": {},- "outputs": [],- "source": [- "# signatures of dspy modules\n",- "class GenerateAnswer(dspy.Signature):\n",- " \"\"\"Answer questions with short factoid answers.\"\"\"\n",- "\n",- " context = dspy.InputField(desc=\"may contain relevant facts\")\n",- " question = dspy.InputField()\n",- " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",- "\n",- "\n",- "class GenerateSearchQuery(dspy.Signature):\n",- " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",- "\n",- " context = dspy.InputField(desc=\"may contain relevant facts\")\n",- " question = dspy.InputField()\n",- " query = dspy.OutputField()"- ]- },- {- "cell_type": "code",- "execution_count": 9,- "metadata": {},- "outputs": [],- "source": [- "class SimplifiedBaleen(dspy.Module):\n",- " def __init__(self, passages_per_hop=2, max_hops=2):\n",- " super().__init__()\n",- "\n",- " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",- " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",- " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",- " self.max_hops = max_hops\n",- "\n",- " def forward(self, question):\n",- " context = []\n",- " for hop in range(self.max_hops):\n",- " query = self.generate_query[hop](context=context, question=question).query\n",- " passages = self.retrieve(query).passages\n",- " context = deduplicate(context + passages)\n",- " pred = self.generate_answer(context=context, question=question)\n",- " pred = dspy.Prediction(context=context, answer=pred.answer)\n",- " return pred"- ]- },- {- "cell_type": "code",- "execution_count": 20,- "metadata": {},- "outputs": [],- "source": [- "class SimplifiedBaleenAssertions(dspy.Module):\n",- " def __init__(self, passages_per_hop=2, max_hops=2):\n",- " super().__init__()\n",- " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",- " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",- " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",- " self.max_hops = max_hops\n",- "\n",- " def forward(self, question):\n",- " context = []\n",- " prev_queries = [question]\n",- "\n",- " for hop in range(self.max_hops):\n",- " query = self.generate_query[hop](context=context, question=question).query\n",- "\n",- " dspy.Suggest(\n",- " len(query) <= 100,\n",- " \"Query should be short and less than 100 characters\",\n",- " )\n",- "\n",- " dspy.Suggest(\n",- " validate_query_distinction_local(prev_queries, query),\n",- " \"Query should be distinct from: \"\n",- " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",- " is_metric=True,\n",- " )\n",- "\n",- " prev_queries.append(query)\n",- " passages = self.retrieve(query).passages\n",- " context = deduplicate(context + passages)\n",- "\n",- " pred = self.generate_answer(context=context, question=question)\n",- " pred = dspy.Prediction(context=context, answer=pred.answer)\n",- " return pred\n"- ]- },- {- "cell_type": "code",- "execution_count": 11,- "metadata": {},- "outputs": [],- "source": [- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"- ]- },- {- "cell_type": "code",- "execution_count": 12,- "metadata": {},- "outputs": [],- "source": [- "def evaluate(module):\n",- " retrieval_score = evaluate_on_hotpotqa(\n",- " module, metric=gold_passages_retrieved\n",- " )\n",- " \n",- " accuracy_score = evaluate_on_hotpotqa(\n",- " module, metric=dspy.evaluate.answer_exact_match\n",- " )\n",- "\n",- " print(f\"## Retrieval Score: {retrieval_score}\")\n",- " print(f\"## Accuracy Score: {accuracy_score}\")"- ]- },- {- "cell_type": "code",- "execution_count": 13,- "metadata": {},- "outputs": [],- "source": [- "# # No Compilation + No Assertion\n",- "# baleen = SimplifiedBaleen()\n",- "# evaluate(baleen)\n"- ]- },- {- "cell_type": "code",- "execution_count": 14,- "metadata": {},- "outputs": [],- "source": [- "# # No Compilation + Yes Assertion\n",- "# baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",- "# evaluate(baleen_with_assertions)"- ]- },- {- "cell_type": "code",- "execution_count": 15,- "metadata": {},- "outputs": [],- "source": [- "# Yes Compilation + No Assertion\n",- "# baleen = SimplifiedBaleen()\n",- "# teleprompter = BootstrapFewShotWithRandomSearch(\n",- "# metric=validate_context_and_answer_and_hops,\n",- "# max_bootstrapped_demos=2,\n",- "# num_candidate_programs=6,\n",- "# )\n",- "\n",- "# compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",- "# evaluate(compiled_baleen)"- ]- },- {- "cell_type": "code",- "execution_count": 21,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Going to sample between 1 and 2 traces per predictor.\n",- "Will attempt to train 6 candidate sets.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 524.07it/s]\n",- "/Users/manishs/Projects/dspy/dspy/evaluate/evaluate.py:130: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",- " df = df.applymap(truncate_cell)\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0%)\n",- "Score: 43.0 for set: [0, 0, 0]\n",- "New best score: 43.0 for seed -3\n",- "Scores so far: [43.0]\n",- "Best score: 43.0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:00<00:00, 514.39it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 42 / 100 (42.0%)\n",- "Score: 42.0 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0]\n",- "Best score: 43.0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%|▏ | 4/300 [00:00<00:00, 1176.85it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 2 full traces after 5 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 4\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:00<00:00, 517.62it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 45 / 100 (45.0%)\n",- "Score: 44.2 for set: [16, 16, 16]\n",- "New best score: 44.2 for seed -1\n",- "Scores so far: [43.0, 42.0, 44.2]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.51\n",- "Average of max per entry across top 3 scores: 0.54\n",- "Average of max per entry across top 5 scores: 0.54\n",- "Average of max per entry across top 8 scores: 0.54\n",- "Average of max per entry across top 9999 scores: 0.54\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 3/300 [00:00<00:00, 1135.03it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 2 full traces after 4 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 4\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:00<00:00, 486.05it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 44 / 100 (44.0%)\n",- "Score: 43.2 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.56\n",- "Average of max per entry across top 8 scores: 0.56\n",- "Average of max per entry across top 9999 scores: 0.56\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 2/300 [00:00<00:00, 1088.72it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 3 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 468.94it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n",- "Score: 40.0 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.58\n",- "Average of max per entry across top 8 scores: 0.58\n",- "Average of max per entry across top 9999 scores: 0.58\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 2/300 [00:00<00:00, 1038.84it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 3 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 2\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:00<00:00, 537.95it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 38 / 100 (38.0%)\n",- "Score: 37.6 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.58\n",- "Average of max per entry across top 8 scores: 0.58\n",- "Average of max per entry across top 9999 scores: 0.58\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%|▏ | 4/300 [00:00<00:00, 1060.71it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 5 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 1\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:00<00:00, 428.19it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 37 / 100 (37.0%)\n",- "Score: 36.8 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.58\n",- "Average of max per entry across top 8 scores: 0.59\n",- "Average of max per entry across top 9999 scores: 0.59\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 3/300 [00:00<00:00, 1050.50it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 1 full traces after 4 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 1\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 555.81it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 43 / 100 (43.0%)\n",- "Score: 42.8 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.57\n",- "Average of max per entry across top 8 scores: 0.6\n",- "Average of max per entry across top 9999 scores: 0.6\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 1%| | 2/300 [00:00<00:00, 616.54it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 2 full traces after 3 examples in round 0.\n",- "# of suggestion failures during bootstrapping: 2\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 460.90it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 40 / 100 (40.0%)\n",- "Score: 39.6 for set: [16, 16, 16]\n",- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8, 39.6]\n",- "Best score: 44.2\n",- "Average of max per entry across top 1 scores: 0.45\n",- "Average of max per entry across top 2 scores: 0.52\n",- "Average of max per entry across top 3 scores: 0.55\n",- "Average of max per entry across top 5 scores: 0.57\n",- "Average of max per entry across top 8 scores: 0.6\n",- "Average of max per entry across top 9999 scores: 0.61\n",- "9 candidate programs found.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 126 / 300 (42.0): 100%|██████████| 300/300 [00:00<00:00, 595.84it/s]\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 126 / 300 (42.0%)\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Average Metric: 150 / 300 (50.0): 100%|██████████| 300/300 [00:00<00:00, 618.44it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Average Metric: 150 / 300 (50.0%)\n",- "## Retrieval Score: 42.0\n",- "## Accuracy Score: 50.0\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"- ]- }- ],- "source": [- "# Yes Compilation + Yes Assertion\n",- "baleen = SimplifiedBaleen()\n",- "teleprompter = BootstrapFewShotWithRandomSearch(\n",- " metric=validate_context_and_answer_and_hops,\n",- " max_bootstrapped_demos=2,\n",- " num_candidate_programs=6,\n",- ")\n",- "compiled_baleen = teleprompter.compile(\n",- " student=assert_transform_module(\n",- " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",- " backtrack_handler,\n",- " ),\n",- " teacher=baleen,\n",- " trainset=trainset,\n",- " valset=devset[:100]\n",- ")\n",- "evaluate(compiled_baleen)"- ]- }- ],- "metadata": {- "kernelspec": {- "display_name": "Python 3",- "language": "python",- "name": "python3"- },- "language_info": {- "codemirror_mode": {- "name": "ipython",- "version": 3- },- "file_extension": ".py",- "mimetype": "text/x-python",- "name": "python",- "nbconvert_exporter": "python",- "pygments_lexer": "ipython3",- "version": "3.11.5"- }- },- "nbformat": 4,- "nbformat_minor": 2-}+ "cells": [+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "import dspy\n",+ "from dsp.utils import deduplicate\n",+ "from dspy.datasets import HotPotQA\n",+ "from dspy.predict.retry import Retry\n",+ "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",+ "from dspy.evaluate.evaluate import Evaluate\n",+ "\n",+ "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "import os\n",+ "import openai\n",+ "openai.api_key = os.getenv('OPENAI_API_KEY')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",+ "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",+ "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",+ "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",+ "devset = [x.with_inputs('question') for x in dataset.dev]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Suggestion helper functions and Teleprompter metric\n",+ "\n",+ "def validate_query_distinction_local(previous_queries, query):\n",+ " \"\"\"check if query is distinct from previous queries\"\"\"\n",+ " if previous_queries == []:\n",+ " return True\n",+ " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",+ " return False\n",+ " return True\n",+ "\n",+ "\n",+ "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",+ " if not dspy.evaluate.answer_exact_match(example, pred):\n",+ " return False\n",+ "\n",+ " if not dspy.evaluate.answer_passage_match(example, pred):\n",+ " return False\n",+ "\n",+ " return True\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Extrinsic metrics\n",+ "\n",+ "def gold_passages_retrieved(example, pred, trace=None):\n",+ " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",+ " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",+ "\n",+ " return gold_titles.issubset(found_titles)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "# signatures of dspy modules\n",+ "class GenerateAnswer(dspy.Signature):\n",+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",+ "\n",+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",+ " question = dspy.InputField()\n",+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",+ "\n",+ "\n",+ "class GenerateSearchQuery(dspy.Signature):\n",+ " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",+ "\n",+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",+ " question = dspy.InputField()\n",+ " query = dspy.OutputField()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "def all_queries_distinct(prev_queries):\n",+ " query_distinct = True\n",+ " for i, query in enumerate(prev_queries):\n",+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",+ " query_distinct = False\n",+ " break\n",+ " return query_distinct"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "class SimplifiedBaleen(dspy.Module):\n",+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",+ " super().__init__()\n",+ "\n",+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",+ " self.max_hops = max_hops\n",+ "\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n",+ " def forward(self, question):\n",+ " context = []\n",+ " prev_queries = [question]\n",+ "\n",+ " for hop in range(self.max_hops):\n",+ " query = self.generate_query[hop](context=context, question=question).query\n",+ " prev_queries.append(query)\n",+ " passages = self.retrieve(query).passages\n",+ " context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n",+ " \n",+ " pred = self.generate_answer(context=context, question=question)\n",+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",+ " return pred"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [],+ "source": [+ "class SimplifiedBaleenAssertions(dspy.Module):\n",+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",+ " super().__init__()\n",+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",+ " self.max_hops = max_hops\n",+ "\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n",+ " def forward(self, question):\n",+ " context = []\n",+ " prev_queries = [question]\n",+ "\n",+ " for hop in range(self.max_hops):\n",+ " query = self.generate_query[hop](context=context, question=question).query\n",+ "\n",+ " dspy.Suggest(\n",+ " len(query) <= 100,\n",+ " \"Query should be short and less than 100 characters\",\n",+ " )\n",+ "\n",+ " dspy.Suggest(\n",+ " validate_query_distinction_local(prev_queries, query),\n",+ " \"Query should be distinct from: \"\n",+ " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",+ " )\n",+ "\n",+ " prev_queries.append(query)\n",+ " passages = self.retrieve(query).passages\n",+ " context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n",+ "\n",+ " pred = self.generate_answer(context=context, question=question)\n",+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",+ " return pred\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [],+ "source": [+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [],+ "source": [+ "def evaluate(module):\n",+ " module.passed_suggestions = 0\n",+ "\n",+ " retrieval_score = evaluate_on_hotpotqa(\n",+ " module, metric=gold_passages_retrieved\n",+ " )\n",+ " \n",+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",+ "\n",+ " accuracy_score = evaluate_on_hotpotqa(\n",+ " module, metric=dspy.evaluate.answer_exact_match\n",+ " )\n",+ "\n",+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",+ "\n",+ " print(f\"## Retrieval Score: {retrieval_score}\")\n",+ " print(f\"## Accuracy Score: {accuracy_score}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# No Compilation + No Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "evaluate(baleen)\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# No Compilation + Yes Assertion\n",+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",+ "evaluate(baleen_with_assertions)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "max_bootstrapped_demos = 2"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Yes Compilation + No Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",+ " metric=validate_context_and_answer_and_hops,\n",+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",+ " num_candidate_programs=6,\n",+ ")\n",+ "\n",+ "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n",+ "evaluate(compiled_baleen)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Yes Compilation + Yes Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",+ " metric=validate_context_and_answer_and_hops,\n",+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",+ " num_candidate_programs=6,\n",+ ")\n",+ "compiled_baleen = teleprompter.compile(\n",+ " student=assert_transform_module(\n",+ " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",+ " suggest_backtrack_handler,\n",+ " ),\n",+ " teacher=baleen,\n",+ " trainset=trainset,\n",+ " valset=devset\n",+ ")\n",+ "evaluate(compiled_baleen)\n"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "Python 3",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.5"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+ }\ No newline at end of file